arena.c revision 9c43c13a35220c10d97a886616899189daceb359
1e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define JEMALLOC_ARENA_C_ 2376b1529a383c39adf4674baf6db83a5e63f97acJason Evans#include "jemalloc/internal/jemalloc_internal.h" 3e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 5e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Data. */ 6e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 7e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t opt_lg_qspace_max = LG_QSPACE_MAX_DEFAULT; 8e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t opt_lg_cspace_max = LG_CSPACE_MAX_DEFAULT; 9e476f8a161d445211fd6e54fe370275196e66bcbJason Evansssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; 10e476f8a161d445211fd6e54fe370275196e66bcbJason Evansuint8_t const *small_size2bin; 1149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansarena_bin_info_t *arena_bin_info; 12e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 13e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Various bin-related settings. */ 14e476f8a161d445211fd6e54fe370275196e66bcbJason Evansunsigned nqbins; 15e476f8a161d445211fd6e54fe370275196e66bcbJason Evansunsigned ncbins; 16e476f8a161d445211fd6e54fe370275196e66bcbJason Evansunsigned nsbins; 17e476f8a161d445211fd6e54fe370275196e66bcbJason Evansunsigned nbins; 18e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t qspace_max; 19e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t cspace_min; 20e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t cspace_max; 21e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t sspace_min; 22e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t sspace_max; 23e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 24e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t lg_mspace; 25e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t mspace_mask; 26e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 27e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 28e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * const_small_size2bin is a static constant lookup table that in the common 2941ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans * case can be used as-is for small_size2bin. 30e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 3141ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans#if (LG_TINY_MIN == 2) 3241ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans#define S2B_4(i) i, 33e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_8(i) S2B_4(i) S2B_4(i) 3441ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans#elif (LG_TINY_MIN == 3) 3541ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans#define S2B_8(i) i, 3641ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans#else 3741ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans# error "Unsupported LG_TINY_MIN" 3841ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans#endif 39e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_16(i) S2B_8(i) S2B_8(i) 40e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_32(i) S2B_16(i) S2B_16(i) 41e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_64(i) S2B_32(i) S2B_32(i) 42e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_128(i) S2B_64(i) S2B_64(i) 43e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_256(i) S2B_128(i) S2B_128(i) 44e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 4541ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans * The number of elements in const_small_size2bin is dependent on the 4641ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans * definition for SUBPAGE. 47e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 4841ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evansstatic JEMALLOC_ATTR(aligned(CACHELINE)) 4941ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans const uint8_t const_small_size2bin[] = { 50e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#if (LG_QUANTUM == 4) 513fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans/* 16-byte quantum **********************/ 52e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_TINY 531e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans# if (LG_TINY_MIN == 2) 543fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans S2B_4(0) /* 4 */ 553fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans S2B_4(1) /* 8 */ 563fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans S2B_8(2) /* 16 */ 571e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans# define S2B_QMIN 2 583fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans# elif (LG_TINY_MIN == 3) 593fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans S2B_8(0) /* 8 */ 603fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans S2B_8(1) /* 16 */ 611e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans# define S2B_QMIN 1 623fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans# else 633fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans# error "Unsupported LG_TINY_MIN" 643fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans# endif 65e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# else 66e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(0) /* 16 */ 67e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define S2B_QMIN 0 68e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 69e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 1) /* 32 */ 70e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 2) /* 48 */ 71e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 3) /* 64 */ 72e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 4) /* 80 */ 73e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 5) /* 96 */ 74e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 6) /* 112 */ 75e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 7) /* 128 */ 76e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define S2B_CMIN (S2B_QMIN + 8) 77e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#else 783fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans/* 8-byte quantum ***********************/ 79e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_TINY 801e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans# if (LG_TINY_MIN == 2) 813fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans S2B_4(0) /* 4 */ 823fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans S2B_4(1) /* 8 */ 831e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans# define S2B_QMIN 1 843fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans# else 853fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans# error "Unsupported LG_TINY_MIN" 863fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans# endif 87e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# else 88e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(0) /* 8 */ 89e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define S2B_QMIN 0 90e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 91e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 1) /* 16 */ 92e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 2) /* 24 */ 93e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 3) /* 32 */ 94e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 4) /* 40 */ 95e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 5) /* 48 */ 96e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 6) /* 56 */ 97e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 7) /* 64 */ 98e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 8) /* 72 */ 99e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 9) /* 80 */ 100e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 10) /* 88 */ 101e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 11) /* 96 */ 102e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 12) /* 104 */ 103e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 13) /* 112 */ 104e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 14) /* 120 */ 105e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 15) /* 128 */ 106e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define S2B_CMIN (S2B_QMIN + 16) 107e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 108e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/****************************************/ 109e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_64(S2B_CMIN + 0) /* 192 */ 110e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_64(S2B_CMIN + 1) /* 256 */ 111e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_64(S2B_CMIN + 2) /* 320 */ 112e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_64(S2B_CMIN + 3) /* 384 */ 113e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_64(S2B_CMIN + 4) /* 448 */ 114e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_64(S2B_CMIN + 5) /* 512 */ 115e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define S2B_SMIN (S2B_CMIN + 6) 116e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 0) /* 768 */ 117e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 1) /* 1024 */ 118e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 2) /* 1280 */ 119e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 3) /* 1536 */ 120e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 4) /* 1792 */ 121e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 5) /* 2048 */ 122e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 6) /* 2304 */ 123e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 7) /* 2560 */ 124e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 8) /* 2816 */ 125e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 9) /* 3072 */ 126e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 10) /* 3328 */ 127e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 11) /* 3584 */ 128e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 12) /* 3840 */ 129e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#if (STATIC_PAGE_SHIFT == 13) 130e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 13) /* 4096 */ 131e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 14) /* 4352 */ 132e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 15) /* 4608 */ 133e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 16) /* 4864 */ 134e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 17) /* 5120 */ 135e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 18) /* 5376 */ 136e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 19) /* 5632 */ 137e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 20) /* 5888 */ 138e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 21) /* 6144 */ 139e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 22) /* 6400 */ 140e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 23) /* 6656 */ 141e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 24) /* 6912 */ 142e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 25) /* 7168 */ 143e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 26) /* 7424 */ 144e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 27) /* 7680 */ 145e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 28) /* 7936 */ 146e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 147e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 148e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_1 149e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_2 150e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_4 151e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_8 152e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_16 153e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_32 154e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_64 155e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_128 156e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_256 157e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_QMIN 158e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_CMIN 159e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_SMIN 160e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 161e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 162e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Function prototypes for non-inline static functions. */ 163e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 164e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_split(arena_t *arena, arena_run_t *run, size_t size, 165e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool large, bool zero); 166e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_chunk_t *arena_chunk_alloc(arena_t *arena); 167e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk); 168e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t *arena_run_alloc(arena_t *arena, size_t size, bool large, 169e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool zero); 1706005f0710cf07d60659d91b20b7ff5592d310027Jason Evansstatic void arena_purge(arena_t *arena, bool all); 171e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty); 172e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, 173e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run, size_t oldsize, size_t newsize); 174e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, 175e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run, size_t oldsize, size_t newsize, bool dirty); 176e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t *arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin); 177e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void *arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin); 178088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evansstatic void arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, 179088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans arena_bin_t *bin); 180e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, 181e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run, arena_bin_t *bin); 182940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansstatic void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, 183940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_run_t *run, arena_bin_t *bin); 184e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, 1858e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans void *ptr, size_t oldsize, size_t size); 186e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, 1878e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans void *ptr, size_t oldsize, size_t size, size_t extra, bool zero); 1888e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansstatic bool arena_ralloc_large(void *ptr, size_t oldsize, size_t size, 1898e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t extra, bool zero); 190e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool small_size2bin_init(void); 191e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 192e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void small_size2bin_validate(void); 193e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 194e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool small_size2bin_init_hard(void); 19549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansstatic size_t bin_info_run_size_calc(arena_bin_info_t *bin_info, 19649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t min_run_size); 19749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansstatic bool bin_info_init(void); 198e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 199e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 200e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 201e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline int 202e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) 203e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 204e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t a_mapelm = (uintptr_t)a; 205e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t b_mapelm = (uintptr_t)b; 206e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 207e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(a != NULL); 208e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(b != NULL); 209e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 210e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return ((a_mapelm > b_mapelm) - (a_mapelm < b_mapelm)); 211e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 212e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 213f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans/* Generate red-black tree functions. */ 214f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evansrb_gen(static JEMALLOC_ATTR(unused), arena_run_tree_, arena_run_tree_t, 21505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_chunk_map_t, u.rb_link, arena_run_comp) 216e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 217e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline int 218e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) 219e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 220e476f8a161d445211fd6e54fe370275196e66bcbJason Evans int ret; 221e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t a_size = a->bits & ~PAGE_MASK; 222e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t b_size = b->bits & ~PAGE_MASK; 223e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2240b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert((a->bits & CHUNK_MAP_KEY) == CHUNK_MAP_KEY || (a->bits & 2250b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans CHUNK_MAP_DIRTY) == (b->bits & CHUNK_MAP_DIRTY)); 22619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 227e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (a_size > b_size) - (a_size < b_size); 228e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == 0) { 229e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t a_mapelm, b_mapelm; 230e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 231e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((a->bits & CHUNK_MAP_KEY) != CHUNK_MAP_KEY) 232e476f8a161d445211fd6e54fe370275196e66bcbJason Evans a_mapelm = (uintptr_t)a; 233e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else { 234e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 235e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Treat keys as though they are lower than anything 236e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * else. 237e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 238e476f8a161d445211fd6e54fe370275196e66bcbJason Evans a_mapelm = 0; 239e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 240e476f8a161d445211fd6e54fe370275196e66bcbJason Evans b_mapelm = (uintptr_t)b; 241e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 242e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm); 243e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 244e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 245e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 246e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 247e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 248f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans/* Generate red-black tree functions. */ 249f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evansrb_gen(static JEMALLOC_ATTR(unused), arena_avail_tree_, arena_avail_tree_t, 25005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_chunk_map_t, u.rb_link, arena_avail_comp) 251e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 252e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline void * 25349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansarena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) 254e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 255e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 25684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans unsigned regind; 25784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 25884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->bitmap_offset); 259e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 26084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans dassert(run->magic == ARENA_RUN_MAGIC); 2611e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans assert(run->nfree > 0); 26284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans assert(bitmap_full(bitmap, &bin_info->bitmap_info) == false); 263e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 26484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans regind = bitmap_sfu(bitmap, &bin_info->bitmap_info); 26584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans ret = (void *)((uintptr_t)run + (uintptr_t)bin_info->reg0_offset + 26684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)(bin_info->reg_size * regind)); 2671e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans run->nfree--; 26884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans if (regind == run->nextind) 26984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans run->nextind++; 27084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans assert(regind < run->nextind); 2711e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans return (ret); 2726109fe07a14b7a619365977d9523db9f8b333792Jason Evans} 2736109fe07a14b7a619365977d9523db9f8b333792Jason Evans 2746109fe07a14b7a619365977d9523db9f8b333792Jason Evansstatic inline void 2751e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evansarena_run_reg_dalloc(arena_run_t *run, void *ptr) 2766109fe07a14b7a619365977d9523db9f8b333792Jason Evans{ 27749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 27849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(chunk->arena, run->bin); 27949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 28084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans unsigned regind = arena_run_regind(run, bin_info, ptr); 28184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 28284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->bitmap_offset); 28384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 28449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(run->nfree < bin_info->nregs); 2851e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans /* Freeing an interior pointer can cause assertion failure. */ 2861e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans assert(((uintptr_t)ptr - ((uintptr_t)run + 28749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans (uintptr_t)bin_info->reg0_offset)) % (uintptr_t)bin_info->reg_size 2881e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans == 0); 28921fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans assert((uintptr_t)ptr >= (uintptr_t)run + 29049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans (uintptr_t)bin_info->reg0_offset); 29184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Freeing an unallocated pointer can cause assertion failure. */ 29284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans assert(bitmap_get(bitmap, &bin_info->bitmap_info, regind)); 293e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 29484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_unset(bitmap, &bin_info->bitmap_info, regind); 2951e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans run->nfree++; 296e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 297e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 29821fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans#ifdef JEMALLOC_DEBUG 29921fb95bba6ea922e0523f269c0d9a32640047a29Jason Evansstatic inline void 30021fb95bba6ea922e0523f269c0d9a32640047a29Jason Evansarena_chunk_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) 30121fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans{ 302d4bab21756279db540866998099522dbd39c05f7Jason Evans size_t i; 30321fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << PAGE_SHIFT)); 304d4bab21756279db540866998099522dbd39c05f7Jason Evans 305d4bab21756279db540866998099522dbd39c05f7Jason Evans for (i = 0; i < PAGE_SIZE / sizeof(size_t); i++) 30621fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans assert(p[i] == 0); 30721fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans} 30821fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans#endif 30921fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans 310e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 311e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, 312e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool zero) 313e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 314e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 315e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t old_ndirty, run_ind, total_pages, need_pages, rem_pages, i; 31619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans size_t flag_dirty; 31719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_t *runs_avail; 318e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 319e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 320e476f8a161d445211fd6e54fe370275196e66bcbJason Evans old_ndirty = chunk->ndirty; 321e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) 322e476f8a161d445211fd6e54fe370275196e66bcbJason Evans >> PAGE_SHIFT); 3237393f44ff025ca67716fc53b68003fd65122fd97Jason Evans flag_dirty = chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY; 32419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans runs_avail = (flag_dirty != 0) ? &arena->runs_avail_dirty : 32519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans &arena->runs_avail_clean; 3267393f44ff025ca67716fc53b68003fd65122fd97Jason Evans total_pages = (chunk->map[run_ind-map_bias].bits & ~PAGE_MASK) >> 327e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT; 3287393f44ff025ca67716fc53b68003fd65122fd97Jason Evans assert((chunk->map[run_ind+total_pages-1-map_bias].bits & 3297393f44ff025ca67716fc53b68003fd65122fd97Jason Evans CHUNK_MAP_DIRTY) == flag_dirty); 330e476f8a161d445211fd6e54fe370275196e66bcbJason Evans need_pages = (size >> PAGE_SHIFT); 331e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(need_pages > 0); 332e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(need_pages <= total_pages); 333e476f8a161d445211fd6e54fe370275196e66bcbJason Evans rem_pages = total_pages - need_pages; 334e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 3357393f44ff025ca67716fc53b68003fd65122fd97Jason Evans arena_avail_tree_remove(runs_avail, &chunk->map[run_ind-map_bias]); 336e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->nactive += need_pages; 337e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 338e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Keep track of trailing unused pages for later use. */ 339e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (rem_pages > 0) { 34019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (flag_dirty != 0) { 3417393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+need_pages-map_bias].bits = 3427393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (rem_pages << PAGE_SHIFT) | CHUNK_MAP_DIRTY; 3437393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+total_pages-1-map_bias].bits = 3447393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (rem_pages << PAGE_SHIFT) | CHUNK_MAP_DIRTY; 34519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } else { 3467393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+need_pages-map_bias].bits = 3477393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (rem_pages << PAGE_SHIFT) | 3487393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[run_ind+need_pages-map_bias].bits & 3493377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans CHUNK_MAP_UNZEROED); 3507393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+total_pages-1-map_bias].bits = 3517393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (rem_pages << PAGE_SHIFT) | 3527393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[run_ind+total_pages-1-map_bias].bits & 3533377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans CHUNK_MAP_UNZEROED); 35419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 35519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_insert(runs_avail, 3567393f44ff025ca67716fc53b68003fd65122fd97Jason Evans &chunk->map[run_ind+need_pages-map_bias]); 357e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 358e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 35919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* Update dirty page accounting. */ 36019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (flag_dirty != 0) { 36119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans chunk->ndirty -= need_pages; 36219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena->ndirty -= need_pages; 36319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 36419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 36519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 36619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Update the page map separately for large vs. small runs, since it is 36719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * possible to avoid iteration for large mallocs. 36819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 36919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (large) { 370e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero) { 37119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (flag_dirty == 0) { 37219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 37319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * The run is clean, so some pages may be 37419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * zeroed (i.e. never before touched). 37519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 37619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans for (i = 0; i < need_pages; i++) { 3777393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if ((chunk->map[run_ind+i-map_bias].bits 3783377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans & CHUNK_MAP_UNZEROED) != 0) { 37919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans memset((void *)((uintptr_t) 38021fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans chunk + ((run_ind+i) << 38119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans PAGE_SHIFT)), 0, 38219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans PAGE_SIZE); 38319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 384940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans#ifdef JEMALLOC_DEBUG 385940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans else { 38621fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans arena_chunk_validate_zeroed( 38721fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans chunk, run_ind+i); 388940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans } 389940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans#endif 39019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 39119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } else { 39219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 39319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * The run is dirty, so all pages must be 39419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * zeroed. 39519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 39619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans memset((void *)((uintptr_t)chunk + (run_ind << 39719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans PAGE_SHIFT)), 0, (need_pages << 39819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans PAGE_SHIFT)); 399e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 400e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 401e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 402e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 40319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Set the last element first, in case the run only contains one 40419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * page (i.e. both statements set the same element). 405e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 4067393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+need_pages-1-map_bias].bits = 4077393f44ff025ca67716fc53b68003fd65122fd97Jason Evans CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED | flag_dirty; 408e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans chunk->map[run_ind-map_bias].bits = size | flag_dirty | 409e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 41019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } else { 41119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans assert(zero == false); 41219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 413940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * Propagate the dirty and unzeroed flags to the allocated 414940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * small run, so that arena_dalloc_bin_run() has the ability to 415940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * conditionally trim clean pages. 41619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 417397e5111b5efd49f61f73c1bad0375c7885a6128Jason Evans chunk->map[run_ind-map_bias].bits = 418397e5111b5efd49f61f73c1bad0375c7885a6128Jason Evans (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_UNZEROED) | 419397e5111b5efd49f61f73c1bad0375c7885a6128Jason Evans CHUNK_MAP_ALLOCATED | flag_dirty; 42021fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans#ifdef JEMALLOC_DEBUG 42121fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans /* 42221fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans * The first page will always be dirtied during small run 42321fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans * initialization, so a validation failure here would not 42421fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans * actually cause an observable failure. 42521fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans */ 42621fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans if (flag_dirty == 0 && 42721fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_UNZEROED) 42821fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans == 0) 42921fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans arena_chunk_validate_zeroed(chunk, run_ind); 43021fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans#endif 43119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans for (i = 1; i < need_pages - 1; i++) { 4327393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+i-map_bias].bits = (i << PAGE_SHIFT) 433397e5111b5efd49f61f73c1bad0375c7885a6128Jason Evans | (chunk->map[run_ind+i-map_bias].bits & 434397e5111b5efd49f61f73c1bad0375c7885a6128Jason Evans CHUNK_MAP_UNZEROED) | CHUNK_MAP_ALLOCATED; 43521fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans#ifdef JEMALLOC_DEBUG 43621fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans if (flag_dirty == 0 && 43721fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans (chunk->map[run_ind+i-map_bias].bits & 43821fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans CHUNK_MAP_UNZEROED) == 0) 43921fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans arena_chunk_validate_zeroed(chunk, run_ind+i); 44021fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans#endif 44119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 442004ed142a66529ecf4a55e8f4fa42ff2e535f586Jason Evans chunk->map[run_ind+need_pages-1-map_bias].bits = ((need_pages 443397e5111b5efd49f61f73c1bad0375c7885a6128Jason Evans - 1) << PAGE_SHIFT) | 444397e5111b5efd49f61f73c1bad0375c7885a6128Jason Evans (chunk->map[run_ind+need_pages-1-map_bias].bits & 445397e5111b5efd49f61f73c1bad0375c7885a6128Jason Evans CHUNK_MAP_UNZEROED) | CHUNK_MAP_ALLOCATED | flag_dirty; 44621fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans#ifdef JEMALLOC_DEBUG 44721fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans if (flag_dirty == 0 && 44821fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans (chunk->map[run_ind+need_pages-1-map_bias].bits & 44921fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans CHUNK_MAP_UNZEROED) == 0) { 45021fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans arena_chunk_validate_zeroed(chunk, 45121fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans run_ind+need_pages-1); 45221fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans } 45321fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans#endif 454e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 455e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 456e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 457e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_chunk_t * 458e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_chunk_alloc(arena_t *arena) 459e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 460e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 461e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t i; 462e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 463e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->spare != NULL) { 46419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_t *runs_avail; 46519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 466e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = arena->spare; 467e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->spare = NULL; 46819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 46919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* Insert the run into the appropriate runs_avail_* tree. */ 4707393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if ((chunk->map[0].bits & CHUNK_MAP_DIRTY) == 0) 47119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans runs_avail = &arena->runs_avail_clean; 47219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans else 47319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans runs_avail = &arena->runs_avail_dirty; 4748de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans assert((chunk->map[0].bits & ~PAGE_MASK) == arena_maxclass); 4758de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans assert((chunk->map[chunk_npages-1-map_bias].bits & ~PAGE_MASK) 4768de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans == arena_maxclass); 4778de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans assert((chunk->map[0].bits & CHUNK_MAP_DIRTY) == 4788de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans (chunk->map[chunk_npages-1-map_bias].bits & 4798de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans CHUNK_MAP_DIRTY)); 4807393f44ff025ca67716fc53b68003fd65122fd97Jason Evans arena_avail_tree_insert(runs_avail, &chunk->map[0]); 481e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 48241631d00618d7262125e501c91d31b4d70e605faJason Evans bool zero; 4833377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans size_t unzeroed; 48441631d00618d7262125e501c91d31b4d70e605faJason Evans 48541631d00618d7262125e501c91d31b4d70e605faJason Evans zero = false; 486e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_unlock(&arena->lock); 4872dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans chunk = (arena_chunk_t *)chunk_alloc(chunksize, false, &zero); 488e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_lock(&arena->lock); 489e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk == NULL) 490e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 491e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 492e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.mapped += chunksize; 493e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 494e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 495e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->arena = arena; 4962caa4715ed4f787f263239ff97dd824636289286Jason Evans ql_elm_new(chunk, link_dirty); 497e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->dirtied = false; 498e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 499e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 500e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Claim that no pages are in use, since the header is merely 501e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * overhead. 502e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 503e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->ndirty = 0; 504e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 505e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 506e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Initialize the map to contain one maximal free untouched run. 50741631d00618d7262125e501c91d31b4d70e605faJason Evans * Mark the pages as zeroed iff chunk_alloc() returned a zeroed 50841631d00618d7262125e501c91d31b4d70e605faJason Evans * chunk. 509e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 5103377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans unzeroed = zero ? 0 : CHUNK_MAP_UNZEROED; 5113377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans chunk->map[0].bits = arena_maxclass | unzeroed; 5123377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans /* 5133377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans * There is no need to initialize the internal page map entries 5143377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans * unless the chunk is not zeroed. 5153377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans */ 5163377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans if (zero == false) { 5173377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans for (i = map_bias+1; i < chunk_npages-1; i++) 5183377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans chunk->map[i-map_bias].bits = unzeroed; 5193377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans } 520940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans#ifdef JEMALLOC_DEBUG 521940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans else { 522940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans for (i = map_bias+1; i < chunk_npages-1; i++) 523940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert(chunk->map[i-map_bias].bits == unzeroed); 524940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans } 525940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans#endif 526004ed142a66529ecf4a55e8f4fa42ff2e535f586Jason Evans chunk->map[chunk_npages-1-map_bias].bits = arena_maxclass | 527004ed142a66529ecf4a55e8f4fa42ff2e535f586Jason Evans unzeroed; 528e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 52919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* Insert the run into the runs_avail_clean tree. */ 53019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_insert(&arena->runs_avail_clean, 5317393f44ff025ca67716fc53b68003fd65122fd97Jason Evans &chunk->map[0]); 53219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 533e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 534e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (chunk); 535e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 536e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 537e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 538e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk) 539e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 54019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_t *runs_avail; 541e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 5428d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans /* 5438d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans * Remove run from the appropriate runs_avail_* tree, so that the arena 5448d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans * does not use it. 5458d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans */ 5467393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if ((chunk->map[0].bits & CHUNK_MAP_DIRTY) == 0) 5478d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans runs_avail = &arena->runs_avail_clean; 5488d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans else 5498d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans runs_avail = &arena->runs_avail_dirty; 5507393f44ff025ca67716fc53b68003fd65122fd97Jason Evans arena_avail_tree_remove(runs_avail, &chunk->map[0]); 5518d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans 5528d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans if (arena->spare != NULL) { 553e00572b384c81bd2aba57fac32f7077a34388915Jason Evans arena_chunk_t *spare = arena->spare; 554e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 5558d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans arena->spare = chunk; 556e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (spare->dirtied) { 557e00572b384c81bd2aba57fac32f7077a34388915Jason Evans ql_remove(&chunk->arena->chunks_dirty, spare, 5582caa4715ed4f787f263239ff97dd824636289286Jason Evans link_dirty); 559e00572b384c81bd2aba57fac32f7077a34388915Jason Evans arena->ndirty -= spare->ndirty; 560e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 561e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_unlock(&arena->lock); 562e00572b384c81bd2aba57fac32f7077a34388915Jason Evans chunk_dealloc((void *)spare, chunksize); 563e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_lock(&arena->lock); 564e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 565e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.mapped -= chunksize; 566e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 5678d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans } else 5688d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans arena->spare = chunk; 569e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 570e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 571e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t * 572e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_alloc(arena_t *arena, size_t size, bool large, bool zero) 573e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 574e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 575e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 576e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *mapelm, key; 577e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 578e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size <= arena_maxclass); 579e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((size & PAGE_MASK) == 0); 580e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 581e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Search the arena's chunks for the lowest best fit. */ 582e476f8a161d445211fd6e54fe370275196e66bcbJason Evans key.bits = size | CHUNK_MAP_KEY; 58319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans mapelm = arena_avail_tree_nsearch(&arena->runs_avail_dirty, &key); 58419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (mapelm != NULL) { 58519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); 5867393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t pageind = (((uintptr_t)mapelm - 5877393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) 5887393f44ff025ca67716fc53b68003fd65122fd97Jason Evans + map_bias; 58919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 59019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << 59119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans PAGE_SHIFT)); 59219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_run_split(arena, run, size, large, zero); 59319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans return (run); 59419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 59519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans mapelm = arena_avail_tree_nsearch(&arena->runs_avail_clean, &key); 596e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mapelm != NULL) { 597e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); 5987393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t pageind = (((uintptr_t)mapelm - 5997393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) 6007393f44ff025ca67716fc53b68003fd65122fd97Jason Evans + map_bias; 601e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 602e00572b384c81bd2aba57fac32f7077a34388915Jason Evans run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << 603e00572b384c81bd2aba57fac32f7077a34388915Jason Evans PAGE_SHIFT)); 604e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_split(arena, run, size, large, zero); 605e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (run); 606e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 607e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 608e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 609e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * No usable runs. Create a new chunk from which to allocate the run. 610e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 611e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = arena_chunk_alloc(arena); 612e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (chunk != NULL) { 6137393f44ff025ca67716fc53b68003fd65122fd97Jason Evans run = (arena_run_t *)((uintptr_t)chunk + (map_bias << 6147393f44ff025ca67716fc53b68003fd65122fd97Jason Evans PAGE_SHIFT)); 615e00572b384c81bd2aba57fac32f7077a34388915Jason Evans arena_run_split(arena, run, size, large, zero); 616e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (run); 617e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 618e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 619e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /* 620e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * arena_chunk_alloc() failed, but another thread may have made 621e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * sufficient memory available while this one dropped arena->lock in 622e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * arena_chunk_alloc(), so search one more time. 623e00572b384c81bd2aba57fac32f7077a34388915Jason Evans */ 62419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans mapelm = arena_avail_tree_nsearch(&arena->runs_avail_dirty, &key); 62519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (mapelm != NULL) { 62619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); 6277393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t pageind = (((uintptr_t)mapelm - 6287393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) 6297393f44ff025ca67716fc53b68003fd65122fd97Jason Evans + map_bias; 63019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 63119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << 63219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans PAGE_SHIFT)); 63319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_run_split(arena, run, size, large, zero); 63419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans return (run); 63519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 63619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans mapelm = arena_avail_tree_nsearch(&arena->runs_avail_clean, &key); 637e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (mapelm != NULL) { 638e00572b384c81bd2aba57fac32f7077a34388915Jason Evans arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); 6397393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t pageind = (((uintptr_t)mapelm - 6407393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) 6417393f44ff025ca67716fc53b68003fd65122fd97Jason Evans + map_bias; 642e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 643e00572b384c81bd2aba57fac32f7077a34388915Jason Evans run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << 644e00572b384c81bd2aba57fac32f7077a34388915Jason Evans PAGE_SHIFT)); 645e00572b384c81bd2aba57fac32f7077a34388915Jason Evans arena_run_split(arena, run, size, large, zero); 646e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (run); 647e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 648e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 649e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (NULL); 650e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 651e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 65205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evansstatic inline void 65305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evansarena_maybe_purge(arena_t *arena) 65405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans{ 65505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 65605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* Enforce opt_lg_dirty_mult. */ 657799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans if (opt_lg_dirty_mult >= 0 && arena->ndirty > arena->npurgatory && 658799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans (arena->ndirty - arena->npurgatory) > chunk_npages && 659799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans (arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty - 660799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory)) 6616005f0710cf07d60659d91b20b7ff5592d310027Jason Evans arena_purge(arena, false); 66205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans} 66305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 66405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evansstatic inline void 66505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evansarena_chunk_purge(arena_t *arena, arena_chunk_t *chunk) 66605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans{ 66705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans ql_head(arena_chunk_map_t) mapelms; 66805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_chunk_map_t *mapelm; 6693377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans size_t pageind, flag_unzeroed; 67005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans#ifdef JEMALLOC_DEBUG 67105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans size_t ndirty; 67205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans#endif 67305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans#ifdef JEMALLOC_STATS 67405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans size_t nmadvise; 67505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans#endif 67605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 67705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans ql_new(&mapelms); 67805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 6793377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans flag_unzeroed = 6802dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED 6812dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans /* 6822dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans * madvise(..., MADV_DONTNEED) results in zero-filled pages for anonymous 6832dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans * mappings, but not for file-backed mappings. 6842dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans */ 6852dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans# ifdef JEMALLOC_SWAP 6863377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans swap_enabled ? CHUNK_MAP_UNZEROED : 6872dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans# endif 6882dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans 0; 6893377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans#else 6903377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans CHUNK_MAP_UNZEROED; 6912dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans#endif 692c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans 69305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* 69405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * If chunk is the spare, temporarily re-allocate it, 1) so that its 69519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * run is reinserted into runs_avail_dirty, and 2) so that it cannot be 69605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * completely discarded by another thread while arena->lock is dropped 69705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * by this thread. Note that the arena_run_dalloc() call will 69805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * implicitly deallocate the chunk, so no explicit action is required 69905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * in this function to deallocate the chunk. 70019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * 70119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Note that once a chunk contains dirty pages, it cannot again contain 70219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * a single run unless 1) it is a dirty run, or 2) this function purges 70319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * dirty pages and causes the transition to a single clean run. Thus 70419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * (chunk == arena->spare) is possible, but it is not possible for 70519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * this function to be called on the spare unless it contains a dirty 70619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * run. 70705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans */ 70819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (chunk == arena->spare) { 7097393f44ff025ca67716fc53b68003fd65122fd97Jason Evans assert((chunk->map[0].bits & CHUNK_MAP_DIRTY) != 0); 71005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_chunk_alloc(arena); 71119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 71205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 71319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* Temporarily allocate all free dirty runs within chunk. */ 7147393f44ff025ca67716fc53b68003fd65122fd97Jason Evans for (pageind = map_bias; pageind < chunk_npages;) { 7157393f44ff025ca67716fc53b68003fd65122fd97Jason Evans mapelm = &chunk->map[pageind-map_bias]; 71605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans if ((mapelm->bits & CHUNK_MAP_ALLOCATED) == 0) { 71719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans size_t npages; 71805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 71919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans npages = mapelm->bits >> PAGE_SHIFT; 720e69bee01de62b56d3e585042d341743239568043Jason Evans assert(pageind + npages <= chunk_npages); 72119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (mapelm->bits & CHUNK_MAP_DIRTY) { 722c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans size_t i; 723c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans 72419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_remove( 72519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans &arena->runs_avail_dirty, mapelm); 726c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans 727940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans mapelm->bits = (npages << PAGE_SHIFT) | 7288de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans flag_unzeroed | CHUNK_MAP_LARGE | 7298de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans CHUNK_MAP_ALLOCATED; 730c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans /* 731c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans * Update internal elements in the page map, so 7323377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans * that CHUNK_MAP_UNZEROED is properly set. 733c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans */ 734c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans for (i = 1; i < npages - 1; i++) { 7357393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[pageind+i-map_bias].bits = 7363377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans flag_unzeroed; 737c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans } 738c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans if (npages > 1) { 7397393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[ 7407393f44ff025ca67716fc53b68003fd65122fd97Jason Evans pageind+npages-1-map_bias].bits = 7418de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans flag_unzeroed | CHUNK_MAP_LARGE | 7428de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans CHUNK_MAP_ALLOCATED; 743c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans } 744c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans 74519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena->nactive += npages; 74619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* Append to list for later processing. */ 74719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans ql_elm_new(mapelm, u.ql_link); 74819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans ql_tail_insert(&mapelms, mapelm, u.ql_link); 74905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 75019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 75105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans pageind += npages; 75205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } else { 75305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* Skip allocated run. */ 754c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans if (mapelm->bits & CHUNK_MAP_LARGE) 75519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans pageind += mapelm->bits >> PAGE_SHIFT; 756c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans else { 75705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_run_t *run = (arena_run_t *)((uintptr_t) 758e69bee01de62b56d3e585042d341743239568043Jason Evans chunk + (uintptr_t)(pageind << PAGE_SHIFT)); 759e69bee01de62b56d3e585042d341743239568043Jason Evans 76019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans assert((mapelm->bits >> PAGE_SHIFT) == 0); 76184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans dassert(run->magic == ARENA_RUN_MAGIC); 76249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(arena, 76349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans run->bin); 76449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = 76549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans &arena_bin_info[binind]; 76649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans pageind += bin_info->run_size >> PAGE_SHIFT; 76705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 76805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 76905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 770e69bee01de62b56d3e585042d341743239568043Jason Evans assert(pageind == chunk_npages); 77105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 77205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans#ifdef JEMALLOC_DEBUG 77305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans ndirty = chunk->ndirty; 77405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans#endif 77505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans#ifdef JEMALLOC_STATS 77605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena->stats.purged += chunk->ndirty; 77705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans#endif 77805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena->ndirty -= chunk->ndirty; 77905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans chunk->ndirty = 0; 78005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans ql_remove(&arena->chunks_dirty, chunk, link_dirty); 78105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans chunk->dirtied = false; 78205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 78305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans malloc_mutex_unlock(&arena->lock); 78405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans#ifdef JEMALLOC_STATS 78505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans nmadvise = 0; 78605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans#endif 78705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans ql_foreach(mapelm, &mapelms, u.ql_link) { 7887393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / 7897393f44ff025ca67716fc53b68003fd65122fd97Jason Evans sizeof(arena_chunk_map_t)) + map_bias; 79019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans size_t npages = mapelm->bits >> PAGE_SHIFT; 79105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 792e69bee01de62b56d3e585042d341743239568043Jason Evans assert(pageind + npages <= chunk_npages); 79305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans#ifdef JEMALLOC_DEBUG 79419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans assert(ndirty >= npages); 79519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans ndirty -= npages; 79605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans#endif 7972dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans 7982dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED 79919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans madvise((void *)((uintptr_t)chunk + (pageind << PAGE_SHIFT)), 80019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans (npages << PAGE_SHIFT), MADV_DONTNEED); 8012dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans#elif defined(JEMALLOC_PURGE_MADVISE_FREE) 8022dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans madvise((void *)((uintptr_t)chunk + (pageind << PAGE_SHIFT)), 8032dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans (npages << PAGE_SHIFT), MADV_FREE); 8042dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans#else 8052dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans# error "No method defined for purging unused dirty pages." 8062dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans#endif 8072dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans 80805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans#ifdef JEMALLOC_STATS 80919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans nmadvise++; 81005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans#endif 81105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 81205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans#ifdef JEMALLOC_DEBUG 81305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans assert(ndirty == 0); 81405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans#endif 81505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans malloc_mutex_lock(&arena->lock); 81605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans#ifdef JEMALLOC_STATS 81705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena->stats.nmadvise += nmadvise; 81805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans#endif 81905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 82005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* Deallocate runs. */ 82105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans for (mapelm = ql_first(&mapelms); mapelm != NULL; 82205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans mapelm = ql_first(&mapelms)) { 8237393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / 8247393f44ff025ca67716fc53b68003fd65122fd97Jason Evans sizeof(arena_chunk_map_t)) + map_bias; 82505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 82605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans (uintptr_t)(pageind << PAGE_SHIFT)); 82705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 82805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans ql_remove(&mapelms, mapelm, u.ql_link); 82905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_run_dalloc(arena, run, false); 83005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 83105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans} 83205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 833e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 8346005f0710cf07d60659d91b20b7ff5592d310027Jason Evansarena_purge(arena_t *arena, bool all) 835e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 836e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 83705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans size_t npurgatory; 838e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 839e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t ndirty = 0; 840e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 8412caa4715ed4f787f263239ff97dd824636289286Jason Evans ql_foreach(chunk, &arena->chunks_dirty, link_dirty) { 8422caa4715ed4f787f263239ff97dd824636289286Jason Evans assert(chunk->dirtied); 8432caa4715ed4f787f263239ff97dd824636289286Jason Evans ndirty += chunk->ndirty; 8442caa4715ed4f787f263239ff97dd824636289286Jason Evans } 845e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ndirty == arena->ndirty); 846e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 847799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans assert(arena->ndirty > arena->npurgatory); 8486005f0710cf07d60659d91b20b7ff5592d310027Jason Evans assert(arena->ndirty > chunk_npages || all); 8496005f0710cf07d60659d91b20b7ff5592d310027Jason Evans assert((arena->nactive >> opt_lg_dirty_mult) < arena->ndirty || all); 850e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 851e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 852e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.npurge++; 853e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 854e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 855e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 85605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * Compute the minimum number of pages that this thread should try to 857799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * purge, and add the result to arena->npurgatory. This will keep 858799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * multiple threads from racing to reduce ndirty below the threshold. 859e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 8606005f0710cf07d60659d91b20b7ff5592d310027Jason Evans npurgatory = arena->ndirty - arena->npurgatory; 8616005f0710cf07d60659d91b20b7ff5592d310027Jason Evans if (all == false) 8626005f0710cf07d60659d91b20b7ff5592d310027Jason Evans npurgatory -= arena->nactive >> opt_lg_dirty_mult; 863799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory += npurgatory; 864799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans 86505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans while (npurgatory > 0) { 86605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* Get next chunk with dirty pages. */ 8672caa4715ed4f787f263239ff97dd824636289286Jason Evans chunk = ql_first(&arena->chunks_dirty); 86805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans if (chunk == NULL) { 86905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* 87005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * This thread was unable to purge as many pages as 87105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * originally intended, due to races with other threads 872799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * that either did some of the purging work, or re-used 873799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * dirty pages. 87405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans */ 875799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory -= npurgatory; 876799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans return; 877e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 87805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans while (chunk->ndirty == 0) { 8792caa4715ed4f787f263239ff97dd824636289286Jason Evans ql_remove(&arena->chunks_dirty, chunk, link_dirty); 880e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->dirtied = false; 88105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans chunk = ql_first(&arena->chunks_dirty); 88205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans if (chunk == NULL) { 88305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* Same logic as for above. */ 884799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory -= npurgatory; 885799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans return; 88605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 887e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 88805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 889799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans if (chunk->ndirty > npurgatory) { 890799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans /* 891799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * This thread will, at a minimum, purge all the dirty 892799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * pages in chunk, so set npurgatory to reflect this 893799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * thread's commitment to purge the pages. This tends 894799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * to reduce the chances of the following scenario: 895799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 896799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 1) This thread sets arena->npurgatory such that 897799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * (arena->ndirty - arena->npurgatory) is at the 898799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * threshold. 899799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 2) This thread drops arena->lock. 900799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 3) Another thread causes one or more pages to be 901799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * dirtied, and immediately determines that it must 902799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * purge dirty pages. 903799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 904799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * If this scenario *does* play out, that's okay, 905799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * because all of the purging work being done really 906799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * needs to happen. 907799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans */ 908799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory += chunk->ndirty - npurgatory; 909799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans npurgatory = chunk->ndirty; 910799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans } 911799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans 912799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory -= chunk->ndirty; 913799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans npurgatory -= chunk->ndirty; 91405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_chunk_purge(arena, chunk); 915e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 916e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 917e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 9186005f0710cf07d60659d91b20b7ff5592d310027Jason Evansvoid 9196005f0710cf07d60659d91b20b7ff5592d310027Jason Evansarena_purge_all(arena_t *arena) 9206005f0710cf07d60659d91b20b7ff5592d310027Jason Evans{ 9216005f0710cf07d60659d91b20b7ff5592d310027Jason Evans 9226005f0710cf07d60659d91b20b7ff5592d310027Jason Evans malloc_mutex_lock(&arena->lock); 9236005f0710cf07d60659d91b20b7ff5592d310027Jason Evans arena_purge(arena, true); 9246005f0710cf07d60659d91b20b7ff5592d310027Jason Evans malloc_mutex_unlock(&arena->lock); 9256005f0710cf07d60659d91b20b7ff5592d310027Jason Evans} 9266005f0710cf07d60659d91b20b7ff5592d310027Jason Evans 927e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 928e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty) 929e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 930e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 93119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans size_t size, run_ind, run_pages, flag_dirty; 93219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_t *runs_avail; 933e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 934e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 935e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) 936e476f8a161d445211fd6e54fe370275196e66bcbJason Evans >> PAGE_SHIFT); 9377393f44ff025ca67716fc53b68003fd65122fd97Jason Evans assert(run_ind >= map_bias); 938e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run_ind < chunk_npages); 9398de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans if ((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_LARGE) != 0) { 9407393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size = chunk->map[run_ind-map_bias].bits & ~PAGE_MASK; 9418de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans assert(size == PAGE_SIZE || 9428de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans (chunk->map[run_ind+(size>>PAGE_SHIFT)-1-map_bias].bits & 9438de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans ~PAGE_MASK) == 0); 9448de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans assert((chunk->map[run_ind+(size>>PAGE_SHIFT)-1-map_bias].bits & 9458de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans CHUNK_MAP_LARGE) != 0); 9468de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans assert((chunk->map[run_ind+(size>>PAGE_SHIFT)-1-map_bias].bits & 9478de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans CHUNK_MAP_ALLOCATED) != 0); 94849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } else { 94949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(arena, run->bin); 95049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 95149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size = bin_info->run_size; 95249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } 953e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_pages = (size >> PAGE_SHIFT); 954e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->nactive -= run_pages; 955e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 95619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 95719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * The run is dirty if the caller claims to have dirtied it, as well as 95819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * if it was already dirty before being allocated. 95919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 9607393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if ((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) != 0) 96119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans dirty = true; 96219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; 96319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans runs_avail = dirty ? &arena->runs_avail_dirty : 96419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans &arena->runs_avail_clean; 96519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 966e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Mark pages as unallocated in the chunk map. */ 967e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (dirty) { 968940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[run_ind-map_bias].bits = size | CHUNK_MAP_DIRTY; 9697393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+run_pages-1-map_bias].bits = size | 970940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_DIRTY; 971e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 97219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans chunk->ndirty += run_pages; 97319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena->ndirty += run_pages; 974e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 9757393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind-map_bias].bits = size | 9763377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_UNZEROED); 9777393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+run_pages-1-map_bias].bits = size | 9787393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[run_ind+run_pages-1-map_bias].bits & 9793377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans CHUNK_MAP_UNZEROED); 980e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 981e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 982e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to coalesce forward. */ 983e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run_ind + run_pages < chunk_npages && 9847393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[run_ind+run_pages-map_bias].bits & CHUNK_MAP_ALLOCATED) 9857393f44ff025ca67716fc53b68003fd65122fd97Jason Evans == 0 && (chunk->map[run_ind+run_pages-map_bias].bits & 9867393f44ff025ca67716fc53b68003fd65122fd97Jason Evans CHUNK_MAP_DIRTY) == flag_dirty) { 9877393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t nrun_size = chunk->map[run_ind+run_pages-map_bias].bits & 988e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ~PAGE_MASK; 98912ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans size_t nrun_pages = nrun_size >> PAGE_SHIFT; 990e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 991e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 992e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Remove successor from runs_avail; the coalesced run is 993e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * inserted later. 994e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 99512ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans assert((chunk->map[run_ind+run_pages+nrun_pages-1-map_bias].bits 99612ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans & ~PAGE_MASK) == nrun_size); 99712ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans assert((chunk->map[run_ind+run_pages+nrun_pages-1-map_bias].bits 99812ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans & CHUNK_MAP_ALLOCATED) == 0); 99912ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans assert((chunk->map[run_ind+run_pages+nrun_pages-1-map_bias].bits 100012ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans & CHUNK_MAP_DIRTY) == flag_dirty); 100119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_remove(runs_avail, 10027393f44ff025ca67716fc53b68003fd65122fd97Jason Evans &chunk->map[run_ind+run_pages-map_bias]); 1003e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1004e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size += nrun_size; 100512ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans run_pages += nrun_pages; 1006e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 10077393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind-map_bias].bits = size | 10087393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_FLAGS_MASK); 10097393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+run_pages-1-map_bias].bits = size | 10107393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[run_ind+run_pages-1-map_bias].bits & 1011e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 1012e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1013e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1014e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to coalesce backward. */ 10157393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if (run_ind > map_bias && (chunk->map[run_ind-1-map_bias].bits & 10167393f44ff025ca67716fc53b68003fd65122fd97Jason Evans CHUNK_MAP_ALLOCATED) == 0 && (chunk->map[run_ind-1-map_bias].bits & 101719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans CHUNK_MAP_DIRTY) == flag_dirty) { 10187393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t prun_size = chunk->map[run_ind-1-map_bias].bits & 10197393f44ff025ca67716fc53b68003fd65122fd97Jason Evans ~PAGE_MASK; 102012ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans size_t prun_pages = prun_size >> PAGE_SHIFT; 1021e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 102212ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans run_ind -= prun_pages; 1023e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1024e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1025e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Remove predecessor from runs_avail; the coalesced run is 1026e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * inserted later. 1027e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 102812ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans assert((chunk->map[run_ind-map_bias].bits & ~PAGE_MASK) 102912ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans == prun_size); 103012ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans assert((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_ALLOCATED) 103112ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans == 0); 103212ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans assert((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) 103312ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans == flag_dirty); 10347393f44ff025ca67716fc53b68003fd65122fd97Jason Evans arena_avail_tree_remove(runs_avail, 10357393f44ff025ca67716fc53b68003fd65122fd97Jason Evans &chunk->map[run_ind-map_bias]); 1036e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1037e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size += prun_size; 103812ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans run_pages += prun_pages; 1039e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 10407393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind-map_bias].bits = size | 10417393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_FLAGS_MASK); 10427393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+run_pages-1-map_bias].bits = size | 10437393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[run_ind+run_pages-1-map_bias].bits & 1044e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 1045e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1046e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1047e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Insert into runs_avail, now that coalescing is complete. */ 10488de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans assert((chunk->map[run_ind-map_bias].bits & ~PAGE_MASK) == 10498de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans (chunk->map[run_ind+run_pages-1-map_bias].bits & ~PAGE_MASK)); 10508de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans assert((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) == 10518de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans (chunk->map[run_ind+run_pages-1-map_bias].bits & CHUNK_MAP_DIRTY)); 10527393f44ff025ca67716fc53b68003fd65122fd97Jason Evans arena_avail_tree_insert(runs_avail, &chunk->map[run_ind-map_bias]); 1053e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 10548d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans if (dirty) { 10558d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans /* 10568d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans * Insert into chunks_dirty before potentially calling 10578d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans * arena_chunk_dealloc(), so that chunks_dirty and 10588d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans * arena->ndirty are consistent. 10598d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans */ 10608d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans if (chunk->dirtied == false) { 10618d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans ql_tail_insert(&arena->chunks_dirty, chunk, link_dirty); 10628d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans chunk->dirtied = true; 10638d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans } 10648d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans } 10658d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans 10664fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans /* 10674fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * Deallocate chunk if it is now completely unused. The bit 10684fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * manipulation checks whether the first run is unallocated and extends 10694fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * to the end of the chunk. 10704fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans */ 10717393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if ((chunk->map[0].bits & (~PAGE_MASK | CHUNK_MAP_ALLOCATED)) == 10727393f44ff025ca67716fc53b68003fd65122fd97Jason Evans arena_maxclass) 1073e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_dealloc(arena, chunk); 1074e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 10754fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans /* 10768d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans * It is okay to do dirty page processing here even if the chunk was 10774fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * deallocated above, since in that case it is the spare. Waiting 10784fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * until after possible chunk deallocation to do dirty processing 10794fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * allows for an old spare to be fully deallocated, thus decreasing the 10804fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * chances of spuriously crossing the dirty page purging threshold. 10814fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans */ 10828d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans if (dirty) 108305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_maybe_purge(arena); 1084e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1085e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1086e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1087e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1088e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t oldsize, size_t newsize) 1089e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1090e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT; 1091e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t head_npages = (oldsize - newsize) >> PAGE_SHIFT; 1092940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans size_t flag_dirty = chunk->map[pageind-map_bias].bits & CHUNK_MAP_DIRTY; 1093e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1094e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(oldsize > newsize); 1095e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1096e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1097e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Update the chunk map so that arena_run_dalloc() can treat the 1098940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * leading run as separately allocated. Set the last element of each 1099940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * run first, in case of single-page runs. 1100e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1101940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_LARGE) != 0); 1102940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_ALLOCATED) != 0); 1103940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind+head_npages-1-map_bias].bits = flag_dirty | 1104940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans (chunk->map[pageind+head_npages-1-map_bias].bits & 1105940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 1106940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind-map_bias].bits = (oldsize - newsize) 1107940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans | flag_dirty | (chunk->map[pageind-map_bias].bits & 1108940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 1109940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1110940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans#ifdef JEMALLOC_DEBUG 1111940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans { 1112940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans size_t tail_npages = newsize >> PAGE_SHIFT; 1113940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] 1114940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans .bits & ~PAGE_MASK) == 0); 1115940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] 1116940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans .bits & CHUNK_MAP_DIRTY) == flag_dirty); 1117940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] 1118940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans .bits & CHUNK_MAP_LARGE) != 0); 1119940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] 1120940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans .bits & CHUNK_MAP_ALLOCATED) != 0); 1121940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans } 1122940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans#endif 1123940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind+head_npages-map_bias].bits = newsize | flag_dirty | 1124940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans (chunk->map[pageind+head_npages-map_bias].bits & 1125940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_FLAGS_MASK) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 1126e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1127e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_dalloc(arena, run, false); 1128e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1129e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1130e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1131e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1132e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t oldsize, size_t newsize, bool dirty) 1133e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1134e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT; 1135940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans size_t head_npages = newsize >> PAGE_SHIFT; 1136940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans size_t tail_npages = (oldsize - newsize) >> PAGE_SHIFT; 1137940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans size_t flag_dirty = chunk->map[pageind-map_bias].bits & 1138940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_DIRTY; 1139e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1140e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(oldsize > newsize); 1141e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1142e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1143e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Update the chunk map so that arena_run_dalloc() can treat the 1144940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * trailing run as separately allocated. Set the last element of each 1145940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * run first, in case of single-page runs. 1146e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1147940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_LARGE) != 0); 1148940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_ALLOCATED) != 0); 1149940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind+head_npages-1-map_bias].bits = flag_dirty | 1150940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans (chunk->map[pageind+head_npages-1-map_bias].bits & 1151940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 1152940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind-map_bias].bits = newsize | flag_dirty | 1153940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans (chunk->map[pageind-map_bias].bits & CHUNK_MAP_UNZEROED) | 1154940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 1155940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1156940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & 1157940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans ~PAGE_MASK) == 0); 1158940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & 1159940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_LARGE) != 0); 1160940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & 1161940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_ALLOCATED) != 0); 1162940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits = 1163940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans flag_dirty | 1164940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans (chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & 1165940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 1166940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind+head_npages-map_bias].bits = (oldsize - newsize) | 1167940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans flag_dirty | (chunk->map[pageind+head_npages-map_bias].bits & 1168940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 1169e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1170e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize), 1171e476f8a161d445211fd6e54fe370275196e66bcbJason Evans dirty); 1172e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1173e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1174e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t * 1175e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) 1176e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1177e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *mapelm; 1178e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 117949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind; 118049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info; 1181e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1182e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Look for a usable run. */ 1183e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapelm = arena_run_tree_first(&bin->runs); 1184e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mapelm != NULL) { 1185e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1186e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind; 1187e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1188e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* run is guaranteed to have available space. */ 1189e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_remove(&bin->runs, mapelm); 1190e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1191e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); 11927393f44ff025ca67716fc53b68003fd65122fd97Jason Evans pageind = ((((uintptr_t)mapelm - (uintptr_t)chunk->map) / 11937393f44ff025ca67716fc53b68003fd65122fd97Jason Evans sizeof(arena_chunk_map_t))) + map_bias; 1194e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 119519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans (mapelm->bits >> PAGE_SHIFT)) 1196e476f8a161d445211fd6e54fe370275196e66bcbJason Evans << PAGE_SHIFT)); 1197e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1198e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->stats.reruns++; 1199e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1200e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (run); 1201e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1202e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* No existing runs have any space available. */ 1203e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 120449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans binind = arena_bin_index(arena, bin); 120549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info = &arena_bin_info[binind]; 120649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 1207e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Allocate a new run. */ 1208e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_unlock(&bin->lock); 1209e69bee01de62b56d3e585042d341743239568043Jason Evans /******************************/ 121086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 121149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans run = arena_run_alloc(arena, bin_info->run_size, false, false); 1212e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (run != NULL) { 121384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 121484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->bitmap_offset); 121584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 1216e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /* Initialize run internals. */ 1217e00572b384c81bd2aba57fac32f7077a34388915Jason Evans run->bin = bin; 121884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans run->nextind = 0; 121949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans run->nfree = bin_info->nregs; 122084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_init(bitmap, &bin_info->bitmap_info); 1221e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 1222e00572b384c81bd2aba57fac32f7077a34388915Jason Evans run->magic = ARENA_RUN_MAGIC; 1223e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1224e69bee01de62b56d3e585042d341743239568043Jason Evans } 1225e69bee01de62b56d3e585042d341743239568043Jason Evans malloc_mutex_unlock(&arena->lock); 1226e69bee01de62b56d3e585042d341743239568043Jason Evans /********************************/ 1227e69bee01de62b56d3e585042d341743239568043Jason Evans malloc_mutex_lock(&bin->lock); 1228e69bee01de62b56d3e585042d341743239568043Jason Evans if (run != NULL) { 1229e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1230e00572b384c81bd2aba57fac32f7077a34388915Jason Evans bin->stats.nruns++; 1231e00572b384c81bd2aba57fac32f7077a34388915Jason Evans bin->stats.curruns++; 1232e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (bin->stats.curruns > bin->stats.highruns) 1233e00572b384c81bd2aba57fac32f7077a34388915Jason Evans bin->stats.highruns = bin->stats.curruns; 1234e00572b384c81bd2aba57fac32f7077a34388915Jason Evans#endif 1235e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (run); 1236e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 1237e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1238e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /* 1239e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * arena_run_alloc() failed, but another thread may have made 1240940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * sufficient memory available while this one dropped bin->lock above, 1241e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * so search one more time. 1242e00572b384c81bd2aba57fac32f7077a34388915Jason Evans */ 1243e00572b384c81bd2aba57fac32f7077a34388915Jason Evans mapelm = arena_run_tree_first(&bin->runs); 1244e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (mapelm != NULL) { 1245e00572b384c81bd2aba57fac32f7077a34388915Jason Evans arena_chunk_t *chunk; 1246e00572b384c81bd2aba57fac32f7077a34388915Jason Evans size_t pageind; 1247e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1248e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /* run is guaranteed to have available space. */ 1249e00572b384c81bd2aba57fac32f7077a34388915Jason Evans arena_run_tree_remove(&bin->runs, mapelm); 1250e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1251e00572b384c81bd2aba57fac32f7077a34388915Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); 12527393f44ff025ca67716fc53b68003fd65122fd97Jason Evans pageind = ((((uintptr_t)mapelm - (uintptr_t)chunk->map) / 12537393f44ff025ca67716fc53b68003fd65122fd97Jason Evans sizeof(arena_chunk_map_t))) + map_bias; 1254e00572b384c81bd2aba57fac32f7077a34388915Jason Evans run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 125519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans (mapelm->bits >> PAGE_SHIFT)) 1256e00572b384c81bd2aba57fac32f7077a34388915Jason Evans << PAGE_SHIFT)); 1257e00572b384c81bd2aba57fac32f7077a34388915Jason Evans#ifdef JEMALLOC_STATS 1258e00572b384c81bd2aba57fac32f7077a34388915Jason Evans bin->stats.reruns++; 1259e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1260e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (run); 1261e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 1262e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1263e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (NULL); 1264e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1265e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 12661e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans/* Re-fill bin->runcur, then call arena_run_reg_alloc(). */ 1267e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void * 1268e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) 1269e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1270e00572b384c81bd2aba57fac32f7077a34388915Jason Evans void *ret; 127149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind; 127249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info; 1273e00572b384c81bd2aba57fac32f7077a34388915Jason Evans arena_run_t *run; 1274e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 127549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans binind = arena_bin_index(arena, bin); 127649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info = &arena_bin_info[binind]; 1277e00572b384c81bd2aba57fac32f7077a34388915Jason Evans bin->runcur = NULL; 1278e00572b384c81bd2aba57fac32f7077a34388915Jason Evans run = arena_bin_nonfull_run_get(arena, bin); 1279e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (bin->runcur != NULL && bin->runcur->nfree > 0) { 1280e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /* 1281e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * Another thread updated runcur while this one ran without the 1282e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * bin lock in arena_bin_nonfull_run_get(). 1283e00572b384c81bd2aba57fac32f7077a34388915Jason Evans */ 128484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans dassert(bin->runcur->magic == ARENA_RUN_MAGIC); 1285e00572b384c81bd2aba57fac32f7077a34388915Jason Evans assert(bin->runcur->nfree > 0); 128649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ret = arena_run_reg_alloc(bin->runcur, bin_info); 1287e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (run != NULL) { 1288940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_chunk_t *chunk; 1289940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1290940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans /* 1291940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * arena_run_alloc() may have allocated run, or it may 129284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * have pulled run from the bin's run tree. Therefore 1293940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * it is unsafe to make any assumptions about how run 1294940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * has previously been used, and arena_bin_lower_run() 1295940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * must be called, as if a region were just deallocated 1296940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * from the run. 1297940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans */ 1298940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 129949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans if (run->nfree == bin_info->nregs) 13008de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_dalloc_bin_run(arena, chunk, run, bin); 13018de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans else 13028de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_bin_lower_run(arena, chunk, run, bin); 1303e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 1304e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (ret); 1305e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 1306e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1307e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (run == NULL) 1308e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1309e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1310e00572b384c81bd2aba57fac32f7077a34388915Jason Evans bin->runcur = run; 1311e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 131284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans dassert(bin->runcur->magic == ARENA_RUN_MAGIC); 1313e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(bin->runcur->nfree > 0); 1314e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 131549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans return (arena_run_reg_alloc(bin->runcur, bin_info)); 1316e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1317e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 131886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans#ifdef JEMALLOC_PROF 131986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evansvoid 132086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evansarena_prof_accum(arena_t *arena, uint64_t accumbytes) 132186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans{ 132286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 132386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (prof_interval != 0) { 132486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena->prof_accumbytes += accumbytes; 132586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (arena->prof_accumbytes >= prof_interval) { 132686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans prof_idump(); 132786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena->prof_accumbytes -= prof_interval; 132886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans } 132986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans } 133086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans} 133186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans#endif 133286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 1333e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TCACHE 1334e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 1335dafde14e08ddfda747aabb2045b350848b601b2eJason Evansarena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind 1336d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans# ifdef JEMALLOC_PROF 1337d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans , uint64_t prof_accumbytes 1338d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans# endif 1339d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans ) 1340e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1341e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i, nfill; 1342e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 1343e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 1344e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ptr; 1345e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1346e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(tbin->ncached == 0); 1347e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1348d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 134986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 1350d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans arena_prof_accum(arena, prof_accumbytes); 135186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&arena->lock); 1352d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 1353e69bee01de62b56d3e585042d341743239568043Jason Evans bin = &arena->bins[binind]; 1354e69bee01de62b56d3e585042d341743239568043Jason Evans malloc_mutex_lock(&bin->lock); 135584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> 1); 135684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans i < nfill; i++) { 1357e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((run = bin->runcur) != NULL && run->nfree > 0) 135849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ptr = arena_run_reg_alloc(run, &arena_bin_info[binind]); 1359e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else 1360e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ptr = arena_bin_malloc_hard(arena, bin); 13613fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans if (ptr == NULL) 1362e476f8a161d445211fd6e54fe370275196e66bcbJason Evans break; 13639c43c13a35220c10d97a886616899189daceb359Jason Evans /* Insert such that low regions get used first. */ 13649c43c13a35220c10d97a886616899189daceb359Jason Evans tbin->avail[nfill - 1 - i] = ptr; 1365e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1366e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 136749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin->stats.allocated += (i - tbin->ncached) * 136849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info[binind].reg_size; 136986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.nmalloc += i; 1370e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->stats.nrequests += tbin->tstats.nrequests; 137186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.nfills++; 1372e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tbin->tstats.nrequests = 0; 1373e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 137486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 1375e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tbin->ncached = i; 1376e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1377e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1378e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1379e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1380e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_malloc_small(arena_t *arena, size_t size, bool zero) 1381e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1382e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1383e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 1384e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 1385e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t binind; 1386e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 138741ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans binind = SMALL_SIZE2BIN(size); 1388dafde14e08ddfda747aabb2045b350848b601b2eJason Evans assert(binind < nbins); 1389e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[binind]; 139049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size = arena_bin_info[binind].reg_size; 1391e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 139286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&bin->lock); 1393e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((run = bin->runcur) != NULL && run->nfree > 0) 139449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ret = arena_run_reg_alloc(run, &arena_bin_info[binind]); 1395e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else 1396e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = arena_bin_malloc_hard(arena, bin); 1397e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1398e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 139986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 1400e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1401e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1402e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1403e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 140486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.allocated += size; 140586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.nmalloc++; 140686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.nrequests++; 1407e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 140886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 1409d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 141086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (isthreaded == false) { 141186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 1412d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans arena_prof_accum(arena, size); 141386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&arena->lock); 141486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans } 1415d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 1416e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1417e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero == false) { 1418e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1419e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 1420e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0xa5, size); 1421e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if (opt_zero) 1422e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 1423e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1424e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else 1425e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 1426e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1427e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1428e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1429e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1430e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1431e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_malloc_large(arena_t *arena, size_t size, bool zero) 1432e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1433e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1434e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1435e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Large allocation. */ 1436e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = PAGE_CEILING(size); 1437e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1438e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (void *)arena_run_alloc(arena, size, true, zero); 1439e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 1440e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1441e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1442e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1443e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1444e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.nmalloc_large++; 1445dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena->stats.nrequests_large++; 1446e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.allocated_large += size; 1447dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++; 1448e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; 1449e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; 1450e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > 1451e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { 1452e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = 1453e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns; 1454e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1455e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1456d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 1457d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans arena_prof_accum(arena, size); 1458d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 1459e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1460e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1461e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero == false) { 1462e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1463e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 1464e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0xa5, size); 1465e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if (opt_zero) 1466e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 1467e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1468e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1469e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1470e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1471e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1472e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1473e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1474e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_malloc(size_t size, bool zero) 1475e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1476e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1477e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size != 0); 1478e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(QUANTUM_CEILING(size) <= arena_maxclass); 1479e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1480dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (size <= small_maxclass) { 1481e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TCACHE 1482e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_t *tcache; 1483e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1484e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((tcache = tcache_get()) != NULL) 1485dafde14e08ddfda747aabb2045b350848b601b2eJason Evans return (tcache_alloc_small(tcache, size, zero)); 1486dafde14e08ddfda747aabb2045b350848b601b2eJason Evans else 1487dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 1488e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1489e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (arena_malloc_small(choose_arena(), size, zero)); 1490dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } else { 1491dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#ifdef JEMALLOC_TCACHE 1492dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (size <= tcache_maxclass) { 1493dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_t *tcache; 1494dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 1495dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if ((tcache = tcache_get()) != NULL) 1496dafde14e08ddfda747aabb2045b350848b601b2eJason Evans return (tcache_alloc_large(tcache, size, zero)); 1497dafde14e08ddfda747aabb2045b350848b601b2eJason Evans else { 1498dafde14e08ddfda747aabb2045b350848b601b2eJason Evans return (arena_malloc_large(choose_arena(), 1499dafde14e08ddfda747aabb2045b350848b601b2eJason Evans size, zero)); 1500dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 1501dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } else 1502dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 1503dafde14e08ddfda747aabb2045b350848b601b2eJason Evans return (arena_malloc_large(choose_arena(), size, zero)); 1504dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 1505e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1506e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1507e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Only handles large allocations that require more than page alignment. */ 1508e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 15098e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansarena_palloc(arena_t *arena, size_t size, size_t alloc_size, size_t alignment, 15108e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans bool zero) 1511e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1512e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1513e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t offset; 1514e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1515e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1516e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((size & PAGE_MASK) == 0); 151793443689a4367cc6fe3de1c9e918adc13d8f9100Jason Evans 151893443689a4367cc6fe3de1c9e918adc13d8f9100Jason Evans alignment = PAGE_CEILING(alignment); 1519e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1520e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 15218e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans ret = (void *)arena_run_alloc(arena, alloc_size, true, zero); 1522e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 1523e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1524e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1525e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1526e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1527e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret); 1528e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1529e476f8a161d445211fd6e54fe370275196e66bcbJason Evans offset = (uintptr_t)ret & (alignment - 1); 1530e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((offset & PAGE_MASK) == 0); 1531e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(offset < alloc_size); 1532e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (offset == 0) 1533e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_tail(arena, chunk, ret, alloc_size, size, false); 1534e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else { 1535e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t leadsize, trailsize; 1536e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1537e476f8a161d445211fd6e54fe370275196e66bcbJason Evans leadsize = alignment - offset; 1538e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (leadsize > 0) { 1539e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_head(arena, chunk, ret, alloc_size, 1540e476f8a161d445211fd6e54fe370275196e66bcbJason Evans alloc_size - leadsize); 1541e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (void *)((uintptr_t)ret + leadsize); 1542e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1543e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1544e476f8a161d445211fd6e54fe370275196e66bcbJason Evans trailsize = alloc_size - leadsize - size; 1545e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (trailsize != 0) { 1546e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Trim trailing space. */ 1547e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(trailsize < alloc_size); 1548e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_tail(arena, chunk, ret, size + trailsize, 1549e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size, false); 1550e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1551e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1552e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1553e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1554e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.nmalloc_large++; 1555dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena->stats.nrequests_large++; 1556e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.allocated_large += size; 1557dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++; 1558e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; 1559e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; 1560e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > 1561e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { 1562e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = 1563e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns; 1564e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1565e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1566e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1567e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1568e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 15698e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (zero == false) { 15708e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (opt_junk) 15718e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans memset(ret, 0xa5, size); 15728e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans else if (opt_zero) 15738e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans memset(ret, 0, size); 15748e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans } 1575e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1576e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1577e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1578e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1579e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Return the size of the allocation pointed to by ptr. */ 1580e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t 1581e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_salloc(const void *ptr) 1582e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1583e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t ret; 1584e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1585e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind, mapbits; 1586e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1587e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ptr != NULL); 1588e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 1589e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1590e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 15917393f44ff025ca67716fc53b68003fd65122fd97Jason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; 15927393f44ff025ca67716fc53b68003fd65122fd97Jason Evans mapbits = chunk->map[pageind-map_bias].bits; 1593e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 1594e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((mapbits & CHUNK_MAP_LARGE) == 0) { 1595e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 159619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans (uintptr_t)((pageind - (mapbits >> PAGE_SHIFT)) << 159719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans PAGE_SHIFT)); 159884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans dassert(run->magic == ARENA_RUN_MAGIC); 159949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(chunk->arena, run->bin); 160049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 1601f00bb7f132e3b74cb36c34223217df0c4394ada4Jason Evans assert(((uintptr_t)ptr - ((uintptr_t)run + 160249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_size == 1603f00bb7f132e3b74cb36c34223217df0c4394ada4Jason Evans 0); 160449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ret = bin_info->reg_size; 1605e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1606f00bb7f132e3b74cb36c34223217df0c4394ada4Jason Evans assert(((uintptr_t)ptr & PAGE_MASK) == 0); 1607e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = mapbits & ~PAGE_MASK; 1608e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ret != 0); 1609e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1610e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1611e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1612e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1613e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 16146109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_PROF 16150b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evansvoid 16160b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evansarena_prof_promoted(const void *ptr, size_t size) 16170b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans{ 16180b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans arena_chunk_t *chunk; 16190b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans size_t pageind, binind; 16200b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 16210b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(ptr != NULL); 16220b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 16230b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(isalloc(ptr) == PAGE_SIZE); 16240b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 16250b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 16267393f44ff025ca67716fc53b68003fd65122fd97Jason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; 162741ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans binind = SMALL_SIZE2BIN(size); 16280b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(binind < nbins); 16297393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[pageind-map_bias].bits = (chunk->map[pageind-map_bias].bits & 1630e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans ~CHUNK_MAP_CLASS_MASK) | ((binind+1) << CHUNK_MAP_CLASS_SHIFT); 16310b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans} 16320b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 16330b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evanssize_t 16340b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evansarena_salloc_demote(const void *ptr) 16350b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans{ 16360b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans size_t ret; 16370b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans arena_chunk_t *chunk; 16380b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans size_t pageind, mapbits; 16390b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 16400b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(ptr != NULL); 16410b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 16420b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 16430b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 16447393f44ff025ca67716fc53b68003fd65122fd97Jason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; 16457393f44ff025ca67716fc53b68003fd65122fd97Jason Evans mapbits = chunk->map[pageind-map_bias].bits; 16460b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 16470b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans if ((mapbits & CHUNK_MAP_LARGE) == 0) { 16480b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 16490b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans (uintptr_t)((pageind - (mapbits >> PAGE_SHIFT)) << 16500b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans PAGE_SHIFT)); 165184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans dassert(run->magic == ARENA_RUN_MAGIC); 165249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(chunk->arena, run->bin); 165349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 16540b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(((uintptr_t)ptr - ((uintptr_t)run + 165549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_size == 16560b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 0); 165749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ret = bin_info->reg_size; 16580b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans } else { 16590b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(((uintptr_t)ptr & PAGE_MASK) == 0); 16600b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans ret = mapbits & ~PAGE_MASK; 16610b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans if (prof_promote && ret == PAGE_SIZE && (mapbits & 1662e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans CHUNK_MAP_CLASS_MASK) != 0) { 16630b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans size_t binind = ((mapbits & CHUNK_MAP_CLASS_MASK) >> 1664e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans CHUNK_MAP_CLASS_SHIFT) - 1; 16650b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(binind < nbins); 166649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ret = arena_bin_info[binind].reg_size; 16670b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans } 16680b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(ret != 0); 16690b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans } 16700b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 16710b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans return (ret); 16720b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans} 16736109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif 16746109fe07a14b7a619365977d9523db9f8b333792Jason Evans 1675e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1676088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evansarena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, 1677e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin) 1678e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1679e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 168019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* Dissociate run from bin. */ 1681e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run == bin->runcur) 1682e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 168349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans else { 168449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(chunk->arena, bin); 168549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 168649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 168749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans if (bin_info->nregs != 1) { 168849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t run_pageind = (((uintptr_t)run - 168949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans (uintptr_t)chunk)) >> PAGE_SHIFT; 169049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_chunk_map_t *run_mapelm = 169149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans &chunk->map[run_pageind-map_bias]; 169249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* 169349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * This block's conditional is necessary because if the 169449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * run only contains one region, then it never gets 169549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * inserted into the non-full runs tree. 169649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 169749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_run_tree_remove(&bin->runs, run_mapelm); 169849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } 1699e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1700088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans} 1701088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans 1702088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evansstatic void 1703088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evansarena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1704088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans arena_bin_t *bin) 1705088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans{ 170649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind; 170749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info; 1708088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans size_t npages, run_ind, past; 1709088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans 1710088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans assert(run != bin->runcur); 1711088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans assert(arena_run_tree_search(&bin->runs, &chunk->map[ 1712088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans (((uintptr_t)run-(uintptr_t)chunk)>>PAGE_SHIFT)-map_bias]) == NULL); 171386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 171449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans binind = arena_bin_index(chunk->arena, run->bin); 171549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info = &arena_bin_info[binind]; 171649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 1717e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_unlock(&bin->lock); 1718e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /******************************/ 171949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans npages = bin_info->run_size >> PAGE_SHIFT; 172019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT); 172184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans past = (size_t)(PAGE_CEILING((uintptr_t)run + 172284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->reg0_offset + (uintptr_t)(run->nextind * 172384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bin_info->reg_size) - (uintptr_t)chunk) >> PAGE_SHIFT); 172486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 172519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 172619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 172719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * If the run was originally clean, and some pages were never touched, 172819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * trim the clean pages before deallocating the dirty portion of the 172919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * run. 173019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 17317393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if ((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) == 0 && past 17327393f44ff025ca67716fc53b68003fd65122fd97Jason Evans - run_ind < npages) { 173319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 173419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Trim clean pages. Convert to large run beforehand. Set the 173519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * last map element first, in case this is a one-page run. 173619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 17377393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+npages-1-map_bias].bits = CHUNK_MAP_LARGE | 1738940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans (chunk->map[run_ind+npages-1-map_bias].bits & 1739940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_FLAGS_MASK); 174049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans chunk->map[run_ind-map_bias].bits = bin_info->run_size | 17417393f44ff025ca67716fc53b68003fd65122fd97Jason Evans CHUNK_MAP_LARGE | (chunk->map[run_ind-map_bias].bits & 17427393f44ff025ca67716fc53b68003fd65122fd97Jason Evans CHUNK_MAP_FLAGS_MASK); 174319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_run_trim_tail(arena, chunk, run, (npages << PAGE_SHIFT), 174421fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans ((past - run_ind) << PAGE_SHIFT), false); 1745940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans /* npages = past - run_ind; */ 17461e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans } 1747e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 1748e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->magic = 0; 1749e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 175019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_run_dalloc(arena, run, true); 175186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&arena->lock); 1752e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /****************************/ 1753e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_lock(&bin->lock); 175486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans#ifdef JEMALLOC_STATS 175586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.curruns--; 175686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans#endif 1757e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1758e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1759940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansstatic void 1760940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansarena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1761940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_bin_t *bin) 1762e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1763e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 17648de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans /* 17658de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans * Make sure that bin->runcur always refers to the lowest non-full run, 17668de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans * if one exists. 17678de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans */ 17688de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans if (bin->runcur == NULL) 17698de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans bin->runcur = run; 17708de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans else if ((uintptr_t)run < (uintptr_t)bin->runcur) { 17718de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans /* Switch runcur. */ 17728de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans if (bin->runcur->nfree > 0) { 17738de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_chunk_t *runcur_chunk = 17748de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans CHUNK_ADDR2BASE(bin->runcur); 17758de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans size_t runcur_pageind = (((uintptr_t)bin->runcur - 17768de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans (uintptr_t)runcur_chunk)) >> PAGE_SHIFT; 17778de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_chunk_map_t *runcur_mapelm = 17788de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans &runcur_chunk->map[runcur_pageind-map_bias]; 17798de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans 17808de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans /* Insert runcur. */ 17818de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_run_tree_insert(&bin->runs, runcur_mapelm); 1782e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 17838de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans bin->runcur = run; 17848de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans } else { 17858de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans size_t run_pageind = (((uintptr_t)run - 17868de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans (uintptr_t)chunk)) >> PAGE_SHIFT; 17878de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_chunk_map_t *run_mapelm = 17888de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans &chunk->map[run_pageind-map_bias]; 17898de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans 17908de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans assert(arena_run_tree_search(&bin->runs, run_mapelm) == NULL); 17918de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_run_tree_insert(&bin->runs, run_mapelm); 1792e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1793940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans} 1794940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1795940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansvoid 1796940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansarena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1797940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_chunk_map_t *mapelm) 1798940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans{ 1799940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans size_t pageind; 1800940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_run_t *run; 1801940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_bin_t *bin; 1802940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans#if (defined(JEMALLOC_FILL) || defined(JEMALLOC_STATS)) 1803940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans size_t size; 1804940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans#endif 1805940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1806940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; 1807940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 1808940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans (mapelm->bits >> PAGE_SHIFT)) << PAGE_SHIFT)); 180984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans dassert(run->magic == ARENA_RUN_MAGIC); 1810940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans bin = run->bin; 181149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(arena, bin); 181249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 1813940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans#if (defined(JEMALLOC_FILL) || defined(JEMALLOC_STATS)) 181449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size = bin_info->reg_size; 1815940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans#endif 1816940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1817940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans#ifdef JEMALLOC_FILL 1818940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans if (opt_junk) 1819940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans memset(ptr, 0x5a, size); 1820940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans#endif 1821940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1822940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_run_reg_dalloc(run, ptr); 182349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans if (run->nfree == bin_info->nregs) { 1824088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans arena_dissociate_bin_run(chunk, run, bin); 18258de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_dalloc_bin_run(arena, chunk, run, bin); 1826088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans } else if (run->nfree == 1 && run != bin->runcur) 18278de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_bin_lower_run(arena, chunk, run, bin); 1828e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1829e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 183086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.allocated -= size; 183186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.ndalloc++; 1832e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1833e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1834e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1835e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1836e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 1837b34e8684ec025aa780997c11f847c19fb269755bJason Evansarena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty, 1838b34e8684ec025aa780997c11f847c19fb269755bJason Evans arena_stats_t *astats, malloc_bin_stats_t *bstats, 1839b34e8684ec025aa780997c11f847c19fb269755bJason Evans malloc_large_stats_t *lstats) 1840b34e8684ec025aa780997c11f847c19fb269755bJason Evans{ 18413c2343518c2b1fbbd66065c75a3c19f908de1d78Jason Evans unsigned i; 1842b34e8684ec025aa780997c11f847c19fb269755bJason Evans 184386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 1844b34e8684ec025aa780997c11f847c19fb269755bJason Evans *nactive += arena->nactive; 1845b34e8684ec025aa780997c11f847c19fb269755bJason Evans *ndirty += arena->ndirty; 1846b34e8684ec025aa780997c11f847c19fb269755bJason Evans 18474201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans astats->mapped += arena->stats.mapped; 1848b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->npurge += arena->stats.npurge; 1849b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->nmadvise += arena->stats.nmadvise; 1850b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->purged += arena->stats.purged; 1851b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->allocated_large += arena->stats.allocated_large; 1852b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->nmalloc_large += arena->stats.nmalloc_large; 1853b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->ndalloc_large += arena->stats.ndalloc_large; 1854dafde14e08ddfda747aabb2045b350848b601b2eJason Evans astats->nrequests_large += arena->stats.nrequests_large; 1855b34e8684ec025aa780997c11f847c19fb269755bJason Evans 18563c2343518c2b1fbbd66065c75a3c19f908de1d78Jason Evans for (i = 0; i < nlclasses; i++) { 1857dafde14e08ddfda747aabb2045b350848b601b2eJason Evans lstats[i].nmalloc += arena->stats.lstats[i].nmalloc; 1858dafde14e08ddfda747aabb2045b350848b601b2eJason Evans lstats[i].ndalloc += arena->stats.lstats[i].ndalloc; 1859b34e8684ec025aa780997c11f847c19fb269755bJason Evans lstats[i].nrequests += arena->stats.lstats[i].nrequests; 1860b34e8684ec025aa780997c11f847c19fb269755bJason Evans lstats[i].highruns += arena->stats.lstats[i].highruns; 1861b34e8684ec025aa780997c11f847c19fb269755bJason Evans lstats[i].curruns += arena->stats.lstats[i].curruns; 1862b34e8684ec025aa780997c11f847c19fb269755bJason Evans } 186386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&arena->lock); 186486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 186586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans for (i = 0; i < nbins; i++) { 186686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena_bin_t *bin = &arena->bins[i]; 186786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 186886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&bin->lock); 186986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].allocated += bin->stats.allocated; 187086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].nmalloc += bin->stats.nmalloc; 187186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].ndalloc += bin->stats.ndalloc; 187286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].nrequests += bin->stats.nrequests; 187386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans#ifdef JEMALLOC_TCACHE 187486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].nfills += bin->stats.nfills; 187586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].nflushes += bin->stats.nflushes; 187686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans#endif 187786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].nruns += bin->stats.nruns; 187886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].reruns += bin->stats.reruns; 187986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].highruns += bin->stats.highruns; 188086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].curruns += bin->stats.curruns; 188186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 188286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans } 1883b34e8684ec025aa780997c11f847c19fb269755bJason Evans} 1884e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1885e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1886e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 1887e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr) 1888e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 188913668262d17fb5950e2441bc9d56a15db9c93877Jason Evans 1890e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Large allocation. */ 1891e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1892990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans# ifndef JEMALLOC_STATS 1893e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 1894990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans# endif 1895e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1896e476f8a161d445211fd6e54fe370275196e66bcbJason Evans { 1897e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#if (defined(JEMALLOC_FILL) || defined(JEMALLOC_STATS)) 1898e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> 1899e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT; 19007393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t size = chunk->map[pageind-map_bias].bits & ~PAGE_MASK; 1901e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1902e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1903e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1904990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans# ifdef JEMALLOC_STATS 1905e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 1906990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans# endif 1907e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ptr, 0x5a, size); 1908e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1909e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1910990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.ndalloc_large++; 1911e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.allocated_large -= size; 1912dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].ndalloc++; 1913e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns--; 1914e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1915e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1916e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1917e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_dalloc(arena, (arena_run_t *)ptr, true); 1918e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1919e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1920e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1921e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, 19228e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t oldsize, size_t size) 1923e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1924e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1925e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size < oldsize); 1926e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1927e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1928e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Shrink the run, and make trailing pages available for other 1929e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * allocations. 1930e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1931e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1932e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size, 1933e476f8a161d445211fd6e54fe370275196e66bcbJason Evans true); 1934e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1935990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.ndalloc_large++; 1936990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.allocated_large -= oldsize; 1937dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].ndalloc++; 1938990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].curruns--; 1939990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans 1940990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.nmalloc_large++; 1941dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena->stats.nrequests_large++; 1942990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.allocated_large += size; 1943dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++; 1944990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; 1945990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; 1946990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > 1947990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { 1948990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = 1949990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns; 1950e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1951e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1952e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1953e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1954e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1955e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 1956e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, 19578e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t oldsize, size_t size, size_t extra, bool zero) 1958e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1959e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; 1960e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t npages = oldsize >> PAGE_SHIFT; 19618e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t followsize; 1962e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 19637393f44ff025ca67716fc53b68003fd65122fd97Jason Evans assert(oldsize == (chunk->map[pageind-map_bias].bits & ~PAGE_MASK)); 1964e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1965e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to extend the run. */ 19668e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans assert(size + extra > oldsize); 1967e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 19687393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if (pageind + npages < chunk_npages && 19697393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[pageind+npages-map_bias].bits 19708e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans & CHUNK_MAP_ALLOCATED) == 0 && (followsize = 19717393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[pageind+npages-map_bias].bits & ~PAGE_MASK) >= size - 19727393f44ff025ca67716fc53b68003fd65122fd97Jason Evans oldsize) { 1973e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1974e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The next run is available and sufficiently large. Split the 1975e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * following run, then merge the first part with the existing 1976e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * allocation. 1977e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1978940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans size_t flag_dirty; 19798e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t splitsize = (oldsize + followsize <= size + extra) 19808e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans ? followsize : size + extra - oldsize; 1981e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_split(arena, (arena_run_t *)((uintptr_t)chunk + 19828e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans ((pageind+npages) << PAGE_SHIFT)), splitsize, true, zero); 1983e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1984088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans size = oldsize + splitsize; 1985088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans npages = size >> PAGE_SHIFT; 1986e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1987940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans /* 1988940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * Mark the extended run as dirty if either portion of the run 1989940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * was dirty before allocation. This is rather pedantic, 1990940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * because there's not actually any sequence of events that 1991940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * could cause the resulting run to be passed to 1992940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * arena_run_dalloc() with the dirty argument set to false 1993940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * (which is when dirty flag consistency would really matter). 1994940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans */ 1995940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans flag_dirty = (chunk->map[pageind-map_bias].bits & 1996940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_DIRTY) | 1997940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans (chunk->map[pageind+npages-1-map_bias].bits & 1998940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_DIRTY); 1999940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind-map_bias].bits = size | flag_dirty 2000940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 2001940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind+npages-1-map_bias].bits = flag_dirty | 2002940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 2003990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans 2004940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans#ifdef JEMALLOC_STATS 2005940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena->stats.ndalloc_large++; 2006940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena->stats.allocated_large -= oldsize; 2007940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].ndalloc++; 2008940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].curruns--; 2009940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 2010940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena->stats.nmalloc_large++; 2011940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena->stats.nrequests_large++; 2012940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena->stats.allocated_large += size; 2013940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++; 2014940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; 2015940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; 2016940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > 2017940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { 2018940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = 2019940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 2020940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1].curruns; 2021940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans } 2022e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2023e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 2024e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 2025e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2026e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 2027e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2028e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 2029e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2030e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2031e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 2032e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Try to resize a large allocation, in order to avoid copying. This will 2033e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * always fail if growing an object, and the following run is already in use. 2034e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 2035e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 20368e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansarena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, 20378e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans bool zero) 2038e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2039e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t psize; 2040e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 20418e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans psize = PAGE_CEILING(size + extra); 2042e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (psize == oldsize) { 2043e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Same size class. */ 2044e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 2045e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk && size < oldsize) { 2046e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - 2047e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size); 2048e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2049e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2050e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 2051e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 2052e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 2053e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_t *arena; 2054e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2055e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 2056e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena = chunk->arena; 205784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans dassert(arena->magic == ARENA_MAGIC); 2058e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2059e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (psize < oldsize) { 2060e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 2061e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Fill before shrinking in order avoid a race. */ 2062e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) { 2063e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + size), 0x5a, 2064e476f8a161d445211fd6e54fe370275196e66bcbJason Evans oldsize - size); 2065e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2066e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 20678e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans arena_ralloc_large_shrink(arena, chunk, ptr, oldsize, 20688e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans psize); 2069e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 2070e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 2071e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool ret = arena_ralloc_large_grow(arena, chunk, ptr, 20728e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans oldsize, PAGE_CEILING(size), 20738e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans psize - PAGE_CEILING(size), zero); 2074e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 20758e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (ret == false && zero == false && opt_zero) { 2076e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + oldsize), 0, 2077e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size - oldsize); 2078e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2079e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2080e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 2081e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2082e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2083e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2084e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2085e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 20868e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansarena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, 20878e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans bool zero) 2088e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2089e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 20908e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* 20918e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * Avoid moving the allocation if the size class can be left the same. 20928e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans */ 2093e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (oldsize <= arena_maxclass) { 2094dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (oldsize <= small_maxclass) { 209549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(arena_bin_info[SMALL_SIZE2BIN(oldsize)].reg_size 209649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans == oldsize); 20978e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if ((size + extra <= small_maxclass && 209841ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans SMALL_SIZE2BIN(size + extra) == 209941ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans SMALL_SIZE2BIN(oldsize)) || (size <= oldsize && 21008e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size + extra >= oldsize)) { 21018e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans#ifdef JEMALLOC_FILL 21028e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (opt_junk && size < oldsize) { 21038e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans memset((void *)((uintptr_t)ptr + size), 21048e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 0x5a, oldsize - size); 21058e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans } 21068e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans#endif 21078e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans return (ptr); 21088e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans } 2109e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 2110e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size <= arena_maxclass); 21118e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (size + extra > small_maxclass) { 21128e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (arena_ralloc_large(ptr, oldsize, size, 21138e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans extra, zero) == false) 2114e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ptr); 2115e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2116e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2117e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2118e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 21198e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* Reallocation would require a move. */ 21208e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans return (NULL); 21218e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans} 21228e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 21238e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansvoid * 21248e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansarena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, 21258e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t alignment, bool zero) 21268e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans{ 21278e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans void *ret; 21288e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t copysize; 21298e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 21308e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* Try to avoid moving the allocation. */ 21318e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans ret = arena_ralloc_no_move(ptr, oldsize, size, extra, zero); 21328e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (ret != NULL) 21338e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans return (ret); 21348e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 21358e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 2136e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 21378e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * size and oldsize are different enough that we need to move the 21388e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * object. In that case, fall back to allocating new space and 21398e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * copying. 2140e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 21418e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (alignment != 0) 21428e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans ret = ipalloc(size + extra, alignment, zero); 21438e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans else 21448e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans ret = arena_malloc(size + extra, zero); 2145e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 21468e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (ret == NULL) { 21478e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (extra == 0) 21488e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans return (NULL); 21498e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* Try again, this time without extra. */ 21508e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (alignment != 0) 21518e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans ret = ipalloc(size, alignment, zero); 21528e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans else 21538e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans ret = arena_malloc(size, zero); 21548e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 21558e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (ret == NULL) 21568e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans return (NULL); 21578e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans } 21588e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 21598e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* Junk/zero-filling were already done by ipalloc()/arena_malloc(). */ 21608e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 21618e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* 21628e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * Copy at most size bytes (not size+extra), since the caller has no 21638e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * expectation that the extra bytes will be reliably preserved. 21648e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans */ 2165e476f8a161d445211fd6e54fe370275196e66bcbJason Evans copysize = (size < oldsize) ? size : oldsize; 2166e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memcpy(ret, ptr, copysize); 2167e476f8a161d445211fd6e54fe370275196e66bcbJason Evans idalloc(ptr); 2168e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 2169e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2170e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2171e476f8a161d445211fd6e54fe370275196e66bcbJason Evansbool 2172e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_new(arena_t *arena, unsigned ind) 2173e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2174e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i; 2175e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 2176e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 21776109fe07a14b7a619365977d9523db9f8b333792Jason Evans arena->ind = ind; 21786109fe07a14b7a619365977d9523db9f8b333792Jason Evans 2179e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (malloc_mutex_init(&arena->lock)) 2180e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 2181e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2182e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 2183e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(&arena->stats, 0, sizeof(arena_stats_t)); 2184dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena->stats.lstats = (malloc_large_stats_t *)base_alloc(nlclasses * 2185dafde14e08ddfda747aabb2045b350848b601b2eJason Evans sizeof(malloc_large_stats_t)); 2186e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->stats.lstats == NULL) 2187e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 2188dafde14e08ddfda747aabb2045b350848b601b2eJason Evans memset(arena->stats.lstats, 0, nlclasses * 2189dafde14e08ddfda747aabb2045b350848b601b2eJason Evans sizeof(malloc_large_stats_t)); 2190e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_TCACHE 2191e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ql_new(&arena->tcache_ql); 2192e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 2193e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2194e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2195d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 2196d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans arena->prof_accumbytes = 0; 2197d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 2198d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans 2199e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Initialize chunks. */ 22002caa4715ed4f787f263239ff97dd824636289286Jason Evans ql_new(&arena->chunks_dirty); 2201e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->spare = NULL; 2202e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2203e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->nactive = 0; 2204e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty = 0; 2205799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory = 0; 2206e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 220719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_new(&arena->runs_avail_clean); 220819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_new(&arena->runs_avail_dirty); 2209e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2210e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Initialize bins. */ 2211e476f8a161d445211fd6e54fe370275196e66bcbJason Evans i = 0; 2212e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TINY 2213e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* (2^n)-spaced tiny bins. */ 2214e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < ntbins; i++) { 2215e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[i]; 221686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (malloc_mutex_init(&bin->lock)) 221786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans return (true); 2218e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 2219e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_new(&bin->runs); 2220e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 2221e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 2222e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2223e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2224e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2225e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2226e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Quantum-spaced bins. */ 2227e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < ntbins + nqbins; i++) { 2228e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[i]; 222986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (malloc_mutex_init(&bin->lock)) 223086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans return (true); 2231e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 2232e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_new(&bin->runs); 2233e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 2234e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 2235e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2236e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2237e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2238e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Cacheline-spaced bins. */ 2239e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < ntbins + nqbins + ncbins; i++) { 2240e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[i]; 224186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (malloc_mutex_init(&bin->lock)) 224286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans return (true); 2243e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 2244e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_new(&bin->runs); 2245e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 2246e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 2247e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2248e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2249e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2250e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Subpage-spaced bins. */ 2251dafde14e08ddfda747aabb2045b350848b601b2eJason Evans for (; i < nbins; i++) { 2252e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[i]; 225386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (malloc_mutex_init(&bin->lock)) 225486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans return (true); 2255e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 2256e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_new(&bin->runs); 2257e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 2258e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 2259e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2260e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2261e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2262e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 2263e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->magic = ARENA_MAGIC; 2264e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2265e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2266e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 2267e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2268e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2269e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 2270e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 2271e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssmall_size2bin_validate(void) 2272e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2273e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t i, size, binind; 2274e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2275e476f8a161d445211fd6e54fe370275196e66bcbJason Evans i = 1; 2276e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_TINY 2277e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Tiny. */ 2278e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < (1U << LG_TINY_MIN); i++) { 2279e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = pow2_ceil(1U << LG_TINY_MIN); 2280e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ffs((int)(size >> (LG_TINY_MIN + 1))); 228141ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans assert(SMALL_SIZE2BIN(i) == binind); 2282e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2283e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < qspace_min; i++) { 2284e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = pow2_ceil(i); 2285e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ffs((int)(size >> (LG_TINY_MIN + 1))); 228641ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans assert(SMALL_SIZE2BIN(i) == binind); 2287e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2288e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 2289e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Quantum-spaced. */ 2290e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i <= qspace_max; i++) { 2291e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = QUANTUM_CEILING(i); 2292e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ntbins + (size >> LG_QUANTUM) - 1; 229341ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans assert(SMALL_SIZE2BIN(i) == binind); 2294e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2295e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Cacheline-spaced. */ 2296e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i <= cspace_max; i++) { 2297e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = CACHELINE_CEILING(i); 2298e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ntbins + nqbins + ((size - cspace_min) >> 2299e476f8a161d445211fd6e54fe370275196e66bcbJason Evans LG_CACHELINE); 230041ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans assert(SMALL_SIZE2BIN(i) == binind); 2301e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2302e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Sub-page. */ 2303e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i <= sspace_max; i++) { 2304e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = SUBPAGE_CEILING(i); 2305e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ntbins + nqbins + ncbins + ((size - sspace_min) 2306e476f8a161d445211fd6e54fe370275196e66bcbJason Evans >> LG_SUBPAGE); 230741ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans assert(SMALL_SIZE2BIN(i) == binind); 2308e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2309e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2310e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2311e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2312e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 2313e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssmall_size2bin_init(void) 2314e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2315e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2316e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_lg_qspace_max != LG_QSPACE_MAX_DEFAULT 2317e476f8a161d445211fd6e54fe370275196e66bcbJason Evans || opt_lg_cspace_max != LG_CSPACE_MAX_DEFAULT 231841ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans || (sizeof(const_small_size2bin) != ((small_maxclass-1) >> 231941ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans LG_TINY_MIN) + 1)) 2320e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (small_size2bin_init_hard()); 2321e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2322e476f8a161d445211fd6e54fe370275196e66bcbJason Evans small_size2bin = const_small_size2bin; 2323e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 2324e476f8a161d445211fd6e54fe370275196e66bcbJason Evans small_size2bin_validate(); 2325e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2326e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 2327e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2328e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2329e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 2330e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssmall_size2bin_init_hard(void) 2331e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2332e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t i, size, binind; 2333e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uint8_t *custom_small_size2bin; 233441ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans#define CUSTOM_SMALL_SIZE2BIN(s) \ 233541ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans custom_small_size2bin[(s-1) >> LG_TINY_MIN] 2336e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2337e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(opt_lg_qspace_max != LG_QSPACE_MAX_DEFAULT 2338e476f8a161d445211fd6e54fe370275196e66bcbJason Evans || opt_lg_cspace_max != LG_CSPACE_MAX_DEFAULT 233941ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans || (sizeof(const_small_size2bin) != ((small_maxclass-1) >> 234041ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans LG_TINY_MIN) + 1)); 2341e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 234241ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans custom_small_size2bin = (uint8_t *) 234341ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans base_alloc(small_maxclass >> LG_TINY_MIN); 2344e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (custom_small_size2bin == NULL) 2345e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 2346e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2347e476f8a161d445211fd6e54fe370275196e66bcbJason Evans i = 1; 2348e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TINY 2349e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Tiny. */ 235041ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans for (; i < (1U << LG_TINY_MIN); i += TINY_MIN) { 2351e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = pow2_ceil(1U << LG_TINY_MIN); 2352e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ffs((int)(size >> (LG_TINY_MIN + 1))); 235341ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans CUSTOM_SMALL_SIZE2BIN(i) = binind; 2354e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 235541ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans for (; i < qspace_min; i += TINY_MIN) { 2356e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = pow2_ceil(i); 2357e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ffs((int)(size >> (LG_TINY_MIN + 1))); 235841ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans CUSTOM_SMALL_SIZE2BIN(i) = binind; 2359e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2360e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2361e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Quantum-spaced. */ 236241ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans for (; i <= qspace_max; i += TINY_MIN) { 2363e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = QUANTUM_CEILING(i); 2364e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ntbins + (size >> LG_QUANTUM) - 1; 236541ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans CUSTOM_SMALL_SIZE2BIN(i) = binind; 2366e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2367e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Cacheline-spaced. */ 236841ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans for (; i <= cspace_max; i += TINY_MIN) { 2369e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = CACHELINE_CEILING(i); 2370e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ntbins + nqbins + ((size - cspace_min) >> 2371e476f8a161d445211fd6e54fe370275196e66bcbJason Evans LG_CACHELINE); 237241ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans CUSTOM_SMALL_SIZE2BIN(i) = binind; 2373e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2374e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Sub-page. */ 237541ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans for (; i <= sspace_max; i += TINY_MIN) { 2376e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = SUBPAGE_CEILING(i); 2377e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ntbins + nqbins + ncbins + ((size - sspace_min) >> 2378e476f8a161d445211fd6e54fe370275196e66bcbJason Evans LG_SUBPAGE); 237941ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans CUSTOM_SMALL_SIZE2BIN(i) = binind; 2380e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2381e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2382e476f8a161d445211fd6e54fe370275196e66bcbJason Evans small_size2bin = custom_small_size2bin; 2383e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 2384e476f8a161d445211fd6e54fe370275196e66bcbJason Evans small_size2bin_validate(); 2385e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2386e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 238741ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans#undef CUSTOM_SMALL_SIZE2BIN 2388e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2389e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 239049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans/* 239149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * Calculate bin_info->run_size such that it meets the following constraints: 239249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * 239349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * *) bin_info->run_size >= min_run_size 239449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * *) bin_info->run_size <= arena_maxclass 239549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed). 239649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * 239784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * bin_info->nregs, bin_info->bitmap_offset, and bin_info->reg0_offset are also 239884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * calculated here, since these settings are all interdependent. 239949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 240049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansstatic size_t 240149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansbin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) 240249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans{ 240349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t try_run_size, good_run_size; 240449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t try_nregs, good_nregs; 240549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t try_hdr_size, good_hdr_size; 240684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans uint32_t try_bitmap_offset, good_bitmap_offset; 240749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans#ifdef JEMALLOC_PROF 240849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t try_ctx0_offset, good_ctx0_offset; 240949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans#endif 241049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t try_reg0_offset, good_reg0_offset; 241149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 241249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(min_run_size >= PAGE_SIZE); 241349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(min_run_size <= arena_maxclass); 241449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 241549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* 241649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * Calculate known-valid settings before entering the run_size 241749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * expansion loop, so that the first part of the loop always copies 241849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * valid settings. 241949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * 242049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * The do..while loop iteratively reduces the number of regions until 242149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * the run header and the regions no longer overlap. A closed formula 242249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * would be quite messy, since there is an interdependency between the 242349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * header's mask length and the number of regions. 242449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 242549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_run_size = min_run_size; 242649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_nregs = ((try_run_size - sizeof(arena_run_t)) / bin_info->reg_size) 242749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans + 1; /* Counter-act try_nregs-- in loop. */ 242849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans do { 242949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_nregs--; 243049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size = sizeof(arena_run_t); 243184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Pad to a long boundary. */ 243284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_hdr_size = LONG_CEILING(try_hdr_size); 243384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_bitmap_offset = try_hdr_size; 243484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Add space for bitmap. */ 243584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_hdr_size += bitmap_size(try_nregs); 243649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans#ifdef JEMALLOC_PROF 243749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans if (opt_prof && prof_promote == false) { 243849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Pad to a quantum boundary. */ 243949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size = QUANTUM_CEILING(try_hdr_size); 244049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_ctx0_offset = try_hdr_size; 244149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Add space for one (prof_ctx_t *) per region. */ 244249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size += try_nregs * sizeof(prof_ctx_t *); 244349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } else 244449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_ctx0_offset = 0; 244549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans#endif 244649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_reg0_offset = try_run_size - (try_nregs * 244749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->reg_size); 244849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } while (try_hdr_size > try_reg0_offset); 244949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 245049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* run_size expansion loop. */ 245149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans do { 245249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* 245349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * Copy valid settings before trying more aggressive settings. 245449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 245549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans good_run_size = try_run_size; 245649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans good_nregs = try_nregs; 245749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans good_hdr_size = try_hdr_size; 245884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans good_bitmap_offset = try_bitmap_offset; 245949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans#ifdef JEMALLOC_PROF 246049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans good_ctx0_offset = try_ctx0_offset; 246149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans#endif 246249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans good_reg0_offset = try_reg0_offset; 246349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 246449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Try more aggressive settings. */ 246549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_run_size += PAGE_SIZE; 246649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_nregs = ((try_run_size - sizeof(arena_run_t)) / 246749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->reg_size) 246849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans + 1; /* Counter-act try_nregs-- in loop. */ 246949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans do { 247049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_nregs--; 247149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size = sizeof(arena_run_t); 247284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Pad to a long boundary. */ 247384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_hdr_size = LONG_CEILING(try_hdr_size); 247484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_bitmap_offset = try_hdr_size; 247584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Add space for bitmap. */ 247684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_hdr_size += bitmap_size(try_nregs); 247749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans#ifdef JEMALLOC_PROF 247849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans if (opt_prof && prof_promote == false) { 247949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Pad to a quantum boundary. */ 248049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size = QUANTUM_CEILING(try_hdr_size); 248149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_ctx0_offset = try_hdr_size; 248249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* 248349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * Add space for one (prof_ctx_t *) per region. 248449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 248549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size += try_nregs * 248649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans sizeof(prof_ctx_t *); 248749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } 248849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans#endif 248949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_reg0_offset = try_run_size - (try_nregs * 249049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->reg_size); 249149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } while (try_hdr_size > try_reg0_offset); 249249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } while (try_run_size <= arena_maxclass 249349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans && try_run_size <= arena_maxclass 249449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans && RUN_MAX_OVRHD * (bin_info->reg_size << 3) > RUN_MAX_OVRHD_RELAX 249549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans && (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size); 249649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 249749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(good_hdr_size <= good_reg0_offset); 249849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 249949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Copy final settings. */ 250049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->run_size = good_run_size; 250149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->nregs = good_nregs; 250284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bin_info->bitmap_offset = good_bitmap_offset; 250349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans#ifdef JEMALLOC_PROF 250449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->ctx0_offset = good_ctx0_offset; 250549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans#endif 250649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->reg0_offset = good_reg0_offset; 250749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 250849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans return (good_run_size); 250949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans} 251049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 251149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansstatic bool 251249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansbin_info_init(void) 251349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans{ 251449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info; 251549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans unsigned i; 251649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t prev_run_size; 251749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 251849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info = base_alloc(sizeof(arena_bin_info_t) * nbins); 251949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans if (arena_bin_info == NULL) 252049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans return (true); 252149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 252249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans prev_run_size = PAGE_SIZE; 252349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans i = 0; 252449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans#ifdef JEMALLOC_TINY 252549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* (2^n)-spaced tiny bins. */ 252649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans for (; i < ntbins; i++) { 252749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info = &arena_bin_info[i]; 252849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->reg_size = (1U << (LG_TINY_MIN + i)); 252949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size); 253084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); 253149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } 253249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans#endif 253349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 253449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Quantum-spaced bins. */ 253549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans for (; i < ntbins + nqbins; i++) { 253649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info = &arena_bin_info[i]; 253749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->reg_size = (i - ntbins + 1) << LG_QUANTUM; 253849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size); 253984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); 254049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } 254149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 254249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Cacheline-spaced bins. */ 254349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans for (; i < ntbins + nqbins + ncbins; i++) { 254449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info = &arena_bin_info[i]; 254549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->reg_size = cspace_min + ((i - (ntbins + nqbins)) << 254649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans LG_CACHELINE); 254749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size); 254884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); 254949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } 255049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 255149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Subpage-spaced bins. */ 255249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans for (; i < nbins; i++) { 255349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info = &arena_bin_info[i]; 255449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->reg_size = sspace_min + ((i - (ntbins + nqbins + 255549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ncbins)) << LG_SUBPAGE); 255649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size); 255784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); 255849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } 255949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 256049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans return (false); 256149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans} 256249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 2563e476f8a161d445211fd6e54fe370275196e66bcbJason Evansbool 2564a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evansarena_boot(void) 2565e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2566a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evans size_t header_size; 25677393f44ff025ca67716fc53b68003fd65122fd97Jason Evans unsigned i; 2568e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2569e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Set variables according to the value of opt_lg_[qc]space_max. */ 2570e476f8a161d445211fd6e54fe370275196e66bcbJason Evans qspace_max = (1U << opt_lg_qspace_max); 2571e476f8a161d445211fd6e54fe370275196e66bcbJason Evans cspace_min = CACHELINE_CEILING(qspace_max); 2572e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (cspace_min == qspace_max) 2573e476f8a161d445211fd6e54fe370275196e66bcbJason Evans cspace_min += CACHELINE; 2574e476f8a161d445211fd6e54fe370275196e66bcbJason Evans cspace_max = (1U << opt_lg_cspace_max); 2575e476f8a161d445211fd6e54fe370275196e66bcbJason Evans sspace_min = SUBPAGE_CEILING(cspace_max); 2576e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (sspace_min == cspace_max) 2577e476f8a161d445211fd6e54fe370275196e66bcbJason Evans sspace_min += SUBPAGE; 2578e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(sspace_min < PAGE_SIZE); 2579e476f8a161d445211fd6e54fe370275196e66bcbJason Evans sspace_max = PAGE_SIZE - SUBPAGE; 2580e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2581e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TINY 2582e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(LG_QUANTUM >= LG_TINY_MIN); 2583e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2584e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ntbins <= LG_QUANTUM); 2585e476f8a161d445211fd6e54fe370275196e66bcbJason Evans nqbins = qspace_max >> LG_QUANTUM; 2586e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ncbins = ((cspace_max - cspace_min) >> LG_CACHELINE) + 1; 2587e476f8a161d445211fd6e54fe370275196e66bcbJason Evans nsbins = ((sspace_max - sspace_min) >> LG_SUBPAGE) + 1; 2588dafde14e08ddfda747aabb2045b350848b601b2eJason Evans nbins = ntbins + nqbins + ncbins + nsbins; 2589e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2590e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 2591e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The small_size2bin lookup table uses uint8_t to encode each bin 2592e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * index, so we cannot support more than 256 small size classes. This 2593e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * limit is difficult to exceed (not even possible with 16B quantum and 2594e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 4KiB pages), and such configurations are impractical, but 2595e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * nonetheless we need to protect against this case in order to avoid 2596e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * undefined behavior. 25970b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * 25980b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * Further constrain nbins to 255 if prof_promote is true, since all 25990b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * small size classes, plus a "not small" size class must be stored in 26000b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * 8 bits of arena_chunk_map_t's bits field. 2601e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 26020b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#ifdef JEMALLOC_PROF 26030b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans if (opt_prof && prof_promote) { 26040b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans if (nbins > 255) { 26050b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans char line_buf[UMAX2S_BUFSIZE]; 26060b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans malloc_write("<jemalloc>: Too many small size classes ("); 2607e73397062ac3ab28a9d377591b63ed19fd154ccaJason Evans malloc_write(u2s(nbins, 10, line_buf)); 26080b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans malloc_write(" > max 255)\n"); 26090b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans abort(); 26100b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans } 26110b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans } else 26120b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#endif 2613dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (nbins > 256) { 2614e476f8a161d445211fd6e54fe370275196e66bcbJason Evans char line_buf[UMAX2S_BUFSIZE]; 2615698805c5252465ddbdcf7048e5ccf16d02533877Jason Evans malloc_write("<jemalloc>: Too many small size classes ("); 2616e73397062ac3ab28a9d377591b63ed19fd154ccaJason Evans malloc_write(u2s(nbins, 10, line_buf)); 2617698805c5252465ddbdcf7048e5ccf16d02533877Jason Evans malloc_write(" > max 256)\n"); 2618e476f8a161d445211fd6e54fe370275196e66bcbJason Evans abort(); 2619e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2620e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2621e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 2622e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Compute the header size such that it is large enough to contain the 26237393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * page map. The page map is biased to omit entries for the header 26247393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * itself, so some iteration is necessary to compute the map bias. 26257393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * 26267393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * 1) Compute safe header_size and map_bias values that include enough 26277393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * space for an unbiased page map. 26287393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * 2) Refine map_bias based on (1) to omit the header pages in the page 26297393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * map. The resulting map_bias may be one too small. 26307393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * 3) Refine map_bias based on (2). The result will be >= the result 26317393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * from (2), and will always be correct. 2632e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 26337393f44ff025ca67716fc53b68003fd65122fd97Jason Evans map_bias = 0; 26347393f44ff025ca67716fc53b68003fd65122fd97Jason Evans for (i = 0; i < 3; i++) { 26357393f44ff025ca67716fc53b68003fd65122fd97Jason Evans header_size = offsetof(arena_chunk_t, map) 26367393f44ff025ca67716fc53b68003fd65122fd97Jason Evans + (sizeof(arena_chunk_map_t) * (chunk_npages-map_bias)); 26377393f44ff025ca67716fc53b68003fd65122fd97Jason Evans map_bias = (header_size >> PAGE_SHIFT) + ((header_size & 26387393f44ff025ca67716fc53b68003fd65122fd97Jason Evans PAGE_MASK) != 0); 26397393f44ff025ca67716fc53b68003fd65122fd97Jason Evans } 26407393f44ff025ca67716fc53b68003fd65122fd97Jason Evans assert(map_bias > 0); 26417393f44ff025ca67716fc53b68003fd65122fd97Jason Evans 26427393f44ff025ca67716fc53b68003fd65122fd97Jason Evans arena_maxclass = chunksize - (map_bias << PAGE_SHIFT); 2643a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evans 264449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans if (small_size2bin_init()) 264549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans return (true); 264649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 264749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans if (bin_info_init()) 264849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans return (true); 264949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 2650a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evans return (false); 2651e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2652