arena.c revision 28b7e42e44a1a77218a941d9dfe5bb643d884219
1#define JEMALLOC_ARENA_C_ 2#include "jemalloc/internal/jemalloc_internal.h" 3 4/******************************************************************************/ 5/* Data. */ 6 7purge_mode_t opt_purge = PURGE_DEFAULT; 8const char *purge_mode_names[] = { 9 "ratio", 10 "decay", 11 "N/A" 12}; 13ssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; 14static ssize_t lg_dirty_mult_default; 15ssize_t opt_decay_time = DECAY_TIME_DEFAULT; 16static ssize_t decay_time_default; 17 18arena_bin_info_t arena_bin_info[NBINS]; 19 20size_t map_bias; 21size_t map_misc_offset; 22size_t arena_maxrun; /* Max run size for arenas. */ 23size_t large_maxclass; /* Max large size class. */ 24unsigned nlclasses; /* Number of large size classes. */ 25unsigned nhclasses; /* Number of huge size classes. */ 26 27/******************************************************************************/ 28/* 29 * Function prototypes for static functions that are referenced prior to 30 * definition. 31 */ 32 33static void arena_chunk_dalloc(tsdn_t *tsdn, arena_t *arena, 34 arena_chunk_t *chunk); 35static void arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena, 36 size_t ndirty_limit); 37static void arena_run_dalloc(tsdn_t *tsdn, arena_t *arena, arena_run_t *run, 38 bool dirty, bool cleaned, bool decommitted); 39static void arena_dalloc_bin_run(tsdn_t *tsdn, arena_t *arena, 40 arena_chunk_t *chunk, arena_run_t *run, arena_bin_t *bin); 41static void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, 42 arena_run_t *run, arena_bin_t *bin); 43 44/******************************************************************************/ 45 46JEMALLOC_INLINE_C size_t 47arena_miscelm_size_get(const arena_chunk_map_misc_t *miscelm) 48{ 49 arena_chunk_t *chunk; 50 size_t pageind, mapbits; 51 52 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); 53 pageind = arena_miscelm_to_pageind(miscelm); 54 mapbits = arena_mapbits_get(chunk, pageind); 55 return (arena_mapbits_size_decode(mapbits)); 56} 57 58JEMALLOC_INLINE_C int 59arena_run_addr_comp(const arena_chunk_map_misc_t *a, 60 const arena_chunk_map_misc_t *b) 61{ 62 uintptr_t a_miscelm = (uintptr_t)a; 63 uintptr_t b_miscelm = (uintptr_t)b; 64 65 assert(a != NULL); 66 assert(b != NULL); 67 68 return ((a_miscelm > b_miscelm) - (a_miscelm < b_miscelm)); 69} 70 71/* Generate pairing heap functions. */ 72ph_gen(static UNUSED, arena_run_heap_, arena_run_heap_t, arena_chunk_map_misc_t, 73 ph_link, arena_run_addr_comp) 74 75#ifdef JEMALLOC_JET 76#undef run_quantize_floor 77#define run_quantize_floor JEMALLOC_N(n_run_quantize_floor) 78#endif 79static size_t 80run_quantize_floor(size_t size) 81{ 82 size_t ret; 83 pszind_t pind; 84 85 assert(size > 0); 86 assert(size <= HUGE_MAXCLASS); 87 assert((size & PAGE_MASK) == 0); 88 89 assert(size != 0); 90 assert(size == PAGE_CEILING(size)); 91 92 pind = psz2ind(size - large_pad + 1); 93 if (pind == 0) { 94 /* 95 * Avoid underflow. This short-circuit would also do the right 96 * thing for all sizes in the range for which there are 97 * PAGE-spaced size classes, but it's simplest to just handle 98 * the one case that would cause erroneous results. 99 */ 100 return (size); 101 } 102 ret = pind2sz(pind - 1) + large_pad; 103 assert(ret <= size); 104 return (ret); 105} 106#ifdef JEMALLOC_JET 107#undef run_quantize_floor 108#define run_quantize_floor JEMALLOC_N(run_quantize_floor) 109run_quantize_t *run_quantize_floor = JEMALLOC_N(n_run_quantize_floor); 110#endif 111 112#ifdef JEMALLOC_JET 113#undef run_quantize_ceil 114#define run_quantize_ceil JEMALLOC_N(n_run_quantize_ceil) 115#endif 116static size_t 117run_quantize_ceil(size_t size) 118{ 119 size_t ret; 120 121 assert(size > 0); 122 assert(size <= HUGE_MAXCLASS); 123 assert((size & PAGE_MASK) == 0); 124 125 ret = run_quantize_floor(size); 126 if (ret < size) { 127 /* 128 * Skip a quantization that may have an adequately large run, 129 * because under-sized runs may be mixed in. This only happens 130 * when an unusual size is requested, i.e. for aligned 131 * allocation, and is just one of several places where linear 132 * search would potentially find sufficiently aligned available 133 * memory somewhere lower. 134 */ 135 ret = pind2sz(psz2ind(ret - large_pad + 1)) + large_pad; 136 } 137 return (ret); 138} 139#ifdef JEMALLOC_JET 140#undef run_quantize_ceil 141#define run_quantize_ceil JEMALLOC_N(run_quantize_ceil) 142run_quantize_t *run_quantize_ceil = JEMALLOC_N(n_run_quantize_ceil); 143#endif 144 145static void 146arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 147 size_t npages) 148{ 149 pszind_t pind = psz2ind(run_quantize_floor(arena_miscelm_size_get( 150 arena_miscelm_get_const(chunk, pageind)))); 151 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 152 LG_PAGE)); 153 arena_run_heap_insert(&arena->runs_avail[pind], 154 arena_miscelm_get_mutable(chunk, pageind)); 155} 156 157static void 158arena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 159 size_t npages) 160{ 161 pszind_t pind = psz2ind(run_quantize_floor(arena_miscelm_size_get( 162 arena_miscelm_get_const(chunk, pageind)))); 163 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 164 LG_PAGE)); 165 arena_run_heap_remove(&arena->runs_avail[pind], 166 arena_miscelm_get_mutable(chunk, pageind)); 167} 168 169static void 170arena_run_dirty_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 171 size_t npages) 172{ 173 arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk, 174 pageind); 175 176 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 177 LG_PAGE)); 178 assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY); 179 assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == 180 CHUNK_MAP_DIRTY); 181 182 qr_new(&miscelm->rd, rd_link); 183 qr_meld(&arena->runs_dirty, &miscelm->rd, rd_link); 184 arena->ndirty += npages; 185} 186 187static void 188arena_run_dirty_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 189 size_t npages) 190{ 191 arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk, 192 pageind); 193 194 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 195 LG_PAGE)); 196 assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY); 197 assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == 198 CHUNK_MAP_DIRTY); 199 200 qr_remove(&miscelm->rd, rd_link); 201 assert(arena->ndirty >= npages); 202 arena->ndirty -= npages; 203} 204 205static size_t 206arena_chunk_dirty_npages(const extent_node_t *node) 207{ 208 209 return (extent_node_size_get(node) >> LG_PAGE); 210} 211 212void 213arena_chunk_cache_maybe_insert(arena_t *arena, extent_node_t *node, bool cache) 214{ 215 216 if (cache) { 217 extent_node_dirty_linkage_init(node); 218 extent_node_dirty_insert(node, &arena->runs_dirty, 219 &arena->chunks_cache); 220 arena->ndirty += arena_chunk_dirty_npages(node); 221 } 222} 223 224void 225arena_chunk_cache_maybe_remove(arena_t *arena, extent_node_t *node, bool dirty) 226{ 227 228 if (dirty) { 229 extent_node_dirty_remove(node); 230 assert(arena->ndirty >= arena_chunk_dirty_npages(node)); 231 arena->ndirty -= arena_chunk_dirty_npages(node); 232 } 233} 234 235JEMALLOC_INLINE_C void * 236arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) 237{ 238 void *ret; 239 size_t regind; 240 arena_chunk_map_misc_t *miscelm; 241 void *rpages; 242 243 assert(run->nfree > 0); 244 assert(!bitmap_full(run->bitmap, &bin_info->bitmap_info)); 245 246 regind = (unsigned)bitmap_sfu(run->bitmap, &bin_info->bitmap_info); 247 miscelm = arena_run_to_miscelm(run); 248 rpages = arena_miscelm_to_rpages(miscelm); 249 ret = (void *)((uintptr_t)rpages + (uintptr_t)bin_info->reg0_offset + 250 (uintptr_t)(bin_info->reg_interval * regind)); 251 run->nfree--; 252 return (ret); 253} 254 255JEMALLOC_INLINE_C void 256arena_run_reg_dalloc(arena_run_t *run, void *ptr) 257{ 258 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 259 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 260 size_t mapbits = arena_mapbits_get(chunk, pageind); 261 szind_t binind = arena_ptr_small_binind_get(ptr, mapbits); 262 arena_bin_info_t *bin_info = &arena_bin_info[binind]; 263 size_t regind = arena_run_regind(run, bin_info, ptr); 264 265 assert(run->nfree < bin_info->nregs); 266 /* Freeing an interior pointer can cause assertion failure. */ 267 assert(((uintptr_t)ptr - 268 ((uintptr_t)arena_miscelm_to_rpages(arena_run_to_miscelm(run)) + 269 (uintptr_t)bin_info->reg0_offset)) % 270 (uintptr_t)bin_info->reg_interval == 0); 271 assert((uintptr_t)ptr >= 272 (uintptr_t)arena_miscelm_to_rpages(arena_run_to_miscelm(run)) + 273 (uintptr_t)bin_info->reg0_offset); 274 /* Freeing an unallocated pointer can cause assertion failure. */ 275 assert(bitmap_get(run->bitmap, &bin_info->bitmap_info, regind)); 276 277 bitmap_unset(run->bitmap, &bin_info->bitmap_info, regind); 278 run->nfree++; 279} 280 281JEMALLOC_INLINE_C void 282arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages) 283{ 284 285 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + 286 (run_ind << LG_PAGE)), (npages << LG_PAGE)); 287 memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0, 288 (npages << LG_PAGE)); 289} 290 291JEMALLOC_INLINE_C void 292arena_run_page_mark_zeroed(arena_chunk_t *chunk, size_t run_ind) 293{ 294 295 JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind 296 << LG_PAGE)), PAGE); 297} 298 299JEMALLOC_INLINE_C void 300arena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) 301{ 302 size_t i; 303 UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE)); 304 305 arena_run_page_mark_zeroed(chunk, run_ind); 306 for (i = 0; i < PAGE / sizeof(size_t); i++) 307 assert(p[i] == 0); 308} 309 310static void 311arena_nactive_add(arena_t *arena, size_t add_pages) 312{ 313 314 if (config_stats) { 315 size_t cactive_add = CHUNK_CEILING((arena->nactive + 316 add_pages) << LG_PAGE) - CHUNK_CEILING(arena->nactive << 317 LG_PAGE); 318 if (cactive_add != 0) 319 stats_cactive_add(cactive_add); 320 } 321 arena->nactive += add_pages; 322} 323 324static void 325arena_nactive_sub(arena_t *arena, size_t sub_pages) 326{ 327 328 if (config_stats) { 329 size_t cactive_sub = CHUNK_CEILING(arena->nactive << LG_PAGE) - 330 CHUNK_CEILING((arena->nactive - sub_pages) << LG_PAGE); 331 if (cactive_sub != 0) 332 stats_cactive_sub(cactive_sub); 333 } 334 arena->nactive -= sub_pages; 335} 336 337static void 338arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, 339 size_t flag_dirty, size_t flag_decommitted, size_t need_pages) 340{ 341 size_t total_pages, rem_pages; 342 343 assert(flag_dirty == 0 || flag_decommitted == 0); 344 345 total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >> 346 LG_PAGE; 347 assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) == 348 flag_dirty); 349 assert(need_pages <= total_pages); 350 rem_pages = total_pages - need_pages; 351 352 arena_avail_remove(arena, chunk, run_ind, total_pages); 353 if (flag_dirty != 0) 354 arena_run_dirty_remove(arena, chunk, run_ind, total_pages); 355 arena_nactive_add(arena, need_pages); 356 357 /* Keep track of trailing unused pages for later use. */ 358 if (rem_pages > 0) { 359 size_t flags = flag_dirty | flag_decommitted; 360 size_t flag_unzeroed_mask = (flags == 0) ? CHUNK_MAP_UNZEROED : 361 0; 362 363 arena_mapbits_unallocated_set(chunk, run_ind+need_pages, 364 (rem_pages << LG_PAGE), flags | 365 (arena_mapbits_unzeroed_get(chunk, run_ind+need_pages) & 366 flag_unzeroed_mask)); 367 arena_mapbits_unallocated_set(chunk, run_ind+total_pages-1, 368 (rem_pages << LG_PAGE), flags | 369 (arena_mapbits_unzeroed_get(chunk, run_ind+total_pages-1) & 370 flag_unzeroed_mask)); 371 if (flag_dirty != 0) { 372 arena_run_dirty_insert(arena, chunk, run_ind+need_pages, 373 rem_pages); 374 } 375 arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages); 376 } 377} 378 379static bool 380arena_run_split_large_helper(arena_t *arena, arena_run_t *run, size_t size, 381 bool remove, bool zero) 382{ 383 arena_chunk_t *chunk; 384 arena_chunk_map_misc_t *miscelm; 385 size_t flag_dirty, flag_decommitted, run_ind, need_pages; 386 size_t flag_unzeroed_mask; 387 388 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 389 miscelm = arena_run_to_miscelm(run); 390 run_ind = arena_miscelm_to_pageind(miscelm); 391 flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); 392 flag_decommitted = arena_mapbits_decommitted_get(chunk, run_ind); 393 need_pages = (size >> LG_PAGE); 394 assert(need_pages > 0); 395 396 if (flag_decommitted != 0 && arena->chunk_hooks.commit(chunk, chunksize, 397 run_ind << LG_PAGE, size, arena->ind)) 398 return (true); 399 400 if (remove) { 401 arena_run_split_remove(arena, chunk, run_ind, flag_dirty, 402 flag_decommitted, need_pages); 403 } 404 405 if (zero) { 406 if (flag_decommitted != 0) { 407 /* The run is untouched, and therefore zeroed. */ 408 JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void 409 *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 410 (need_pages << LG_PAGE)); 411 } else if (flag_dirty != 0) { 412 /* The run is dirty, so all pages must be zeroed. */ 413 arena_run_zero(chunk, run_ind, need_pages); 414 } else { 415 /* 416 * The run is clean, so some pages may be zeroed (i.e. 417 * never before touched). 418 */ 419 size_t i; 420 for (i = 0; i < need_pages; i++) { 421 if (arena_mapbits_unzeroed_get(chunk, run_ind+i) 422 != 0) 423 arena_run_zero(chunk, run_ind+i, 1); 424 else if (config_debug) { 425 arena_run_page_validate_zeroed(chunk, 426 run_ind+i); 427 } else { 428 arena_run_page_mark_zeroed(chunk, 429 run_ind+i); 430 } 431 } 432 } 433 } else { 434 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + 435 (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); 436 } 437 438 /* 439 * Set the last element first, in case the run only contains one page 440 * (i.e. both statements set the same element). 441 */ 442 flag_unzeroed_mask = (flag_dirty | flag_decommitted) == 0 ? 443 CHUNK_MAP_UNZEROED : 0; 444 arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0, flag_dirty | 445 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, 446 run_ind+need_pages-1))); 447 arena_mapbits_large_set(chunk, run_ind, size, flag_dirty | 448 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, run_ind))); 449 return (false); 450} 451 452static bool 453arena_run_split_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) 454{ 455 456 return (arena_run_split_large_helper(arena, run, size, true, zero)); 457} 458 459static bool 460arena_run_init_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) 461{ 462 463 return (arena_run_split_large_helper(arena, run, size, false, zero)); 464} 465 466static bool 467arena_run_split_small(arena_t *arena, arena_run_t *run, size_t size, 468 szind_t binind) 469{ 470 arena_chunk_t *chunk; 471 arena_chunk_map_misc_t *miscelm; 472 size_t flag_dirty, flag_decommitted, run_ind, need_pages, i; 473 474 assert(binind != BININD_INVALID); 475 476 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 477 miscelm = arena_run_to_miscelm(run); 478 run_ind = arena_miscelm_to_pageind(miscelm); 479 flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); 480 flag_decommitted = arena_mapbits_decommitted_get(chunk, run_ind); 481 need_pages = (size >> LG_PAGE); 482 assert(need_pages > 0); 483 484 if (flag_decommitted != 0 && arena->chunk_hooks.commit(chunk, chunksize, 485 run_ind << LG_PAGE, size, arena->ind)) 486 return (true); 487 488 arena_run_split_remove(arena, chunk, run_ind, flag_dirty, 489 flag_decommitted, need_pages); 490 491 for (i = 0; i < need_pages; i++) { 492 size_t flag_unzeroed = arena_mapbits_unzeroed_get(chunk, 493 run_ind+i); 494 arena_mapbits_small_set(chunk, run_ind+i, i, binind, 495 flag_unzeroed); 496 if (config_debug && flag_dirty == 0 && flag_unzeroed == 0) 497 arena_run_page_validate_zeroed(chunk, run_ind+i); 498 } 499 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + 500 (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); 501 return (false); 502} 503 504static arena_chunk_t * 505arena_chunk_init_spare(arena_t *arena) 506{ 507 arena_chunk_t *chunk; 508 509 assert(arena->spare != NULL); 510 511 chunk = arena->spare; 512 arena->spare = NULL; 513 514 assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); 515 assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); 516 assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == 517 arena_maxrun); 518 assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == 519 arena_maxrun); 520 assert(arena_mapbits_dirty_get(chunk, map_bias) == 521 arena_mapbits_dirty_get(chunk, chunk_npages-1)); 522 523 return (chunk); 524} 525 526static bool 527arena_chunk_register(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, 528 bool zero) 529{ 530 531 /* 532 * The extent node notion of "committed" doesn't directly apply to 533 * arena chunks. Arbitrarily mark them as committed. The commit state 534 * of runs is tracked individually, and upon chunk deallocation the 535 * entire chunk is in a consistent commit state. 536 */ 537 extent_node_init(&chunk->node, arena, chunk, chunksize, zero, true); 538 extent_node_achunk_set(&chunk->node, true); 539 return (chunk_register(tsdn, chunk, &chunk->node)); 540} 541 542static arena_chunk_t * 543arena_chunk_alloc_internal_hard(tsdn_t *tsdn, arena_t *arena, 544 chunk_hooks_t *chunk_hooks, bool *zero, bool *commit) 545{ 546 arena_chunk_t *chunk; 547 548 malloc_mutex_unlock(tsdn, &arena->lock); 549 550 chunk = (arena_chunk_t *)chunk_alloc_wrapper(tsdn, arena, chunk_hooks, 551 NULL, chunksize, chunksize, zero, commit); 552 if (chunk != NULL && !*commit) { 553 /* Commit header. */ 554 if (chunk_hooks->commit(chunk, chunksize, 0, map_bias << 555 LG_PAGE, arena->ind)) { 556 chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, 557 (void *)chunk, chunksize, *zero, *commit); 558 chunk = NULL; 559 } 560 } 561 if (chunk != NULL && arena_chunk_register(tsdn, arena, chunk, *zero)) { 562 if (!*commit) { 563 /* Undo commit of header. */ 564 chunk_hooks->decommit(chunk, chunksize, 0, map_bias << 565 LG_PAGE, arena->ind); 566 } 567 chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, (void *)chunk, 568 chunksize, *zero, *commit); 569 chunk = NULL; 570 } 571 572 malloc_mutex_lock(tsdn, &arena->lock); 573 return (chunk); 574} 575 576static arena_chunk_t * 577arena_chunk_alloc_internal(tsdn_t *tsdn, arena_t *arena, bool *zero, 578 bool *commit) 579{ 580 arena_chunk_t *chunk; 581 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; 582 583 chunk = chunk_alloc_cache(tsdn, arena, &chunk_hooks, NULL, chunksize, 584 chunksize, zero, commit, true); 585 if (chunk != NULL) { 586 if (arena_chunk_register(tsdn, arena, chunk, *zero)) { 587 chunk_dalloc_cache(tsdn, arena, &chunk_hooks, chunk, 588 chunksize, true); 589 return (NULL); 590 } 591 } 592 if (chunk == NULL) { 593 chunk = arena_chunk_alloc_internal_hard(tsdn, arena, 594 &chunk_hooks, zero, commit); 595 } 596 597 if (config_stats && chunk != NULL) { 598 arena->stats.mapped += chunksize; 599 arena->stats.metadata_mapped += (map_bias << LG_PAGE); 600 } 601 602 return (chunk); 603} 604 605static arena_chunk_t * 606arena_chunk_init_hard(tsdn_t *tsdn, arena_t *arena) 607{ 608 arena_chunk_t *chunk; 609 bool zero, commit; 610 size_t flag_unzeroed, flag_decommitted, i; 611 612 assert(arena->spare == NULL); 613 614 zero = false; 615 commit = false; 616 chunk = arena_chunk_alloc_internal(tsdn, arena, &zero, &commit); 617 if (chunk == NULL) 618 return (NULL); 619 620 /* 621 * Initialize the map to contain one maximal free untouched run. Mark 622 * the pages as zeroed if arena_chunk_alloc_internal() returned a zeroed 623 * or decommitted chunk. 624 */ 625 flag_unzeroed = (zero || !commit) ? 0 : CHUNK_MAP_UNZEROED; 626 flag_decommitted = commit ? 0 : CHUNK_MAP_DECOMMITTED; 627 arena_mapbits_unallocated_set(chunk, map_bias, arena_maxrun, 628 flag_unzeroed | flag_decommitted); 629 /* 630 * There is no need to initialize the internal page map entries unless 631 * the chunk is not zeroed. 632 */ 633 if (!zero) { 634 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED( 635 (void *)arena_bitselm_get_const(chunk, map_bias+1), 636 (size_t)((uintptr_t)arena_bitselm_get_const(chunk, 637 chunk_npages-1) - 638 (uintptr_t)arena_bitselm_get_const(chunk, map_bias+1))); 639 for (i = map_bias+1; i < chunk_npages-1; i++) 640 arena_mapbits_internal_set(chunk, i, flag_unzeroed); 641 } else { 642 JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void 643 *)arena_bitselm_get_const(chunk, map_bias+1), 644 (size_t)((uintptr_t)arena_bitselm_get_const(chunk, 645 chunk_npages-1) - 646 (uintptr_t)arena_bitselm_get_const(chunk, map_bias+1))); 647 if (config_debug) { 648 for (i = map_bias+1; i < chunk_npages-1; i++) { 649 assert(arena_mapbits_unzeroed_get(chunk, i) == 650 flag_unzeroed); 651 } 652 } 653 } 654 arena_mapbits_unallocated_set(chunk, chunk_npages-1, arena_maxrun, 655 flag_unzeroed); 656 657 return (chunk); 658} 659 660static arena_chunk_t * 661arena_chunk_alloc(tsdn_t *tsdn, arena_t *arena) 662{ 663 arena_chunk_t *chunk; 664 665 if (arena->spare != NULL) 666 chunk = arena_chunk_init_spare(arena); 667 else { 668 chunk = arena_chunk_init_hard(tsdn, arena); 669 if (chunk == NULL) 670 return (NULL); 671 } 672 673 ql_elm_new(&chunk->node, ql_link); 674 ql_tail_insert(&arena->achunks, &chunk->node, ql_link); 675 arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias); 676 677 return (chunk); 678} 679 680static void 681arena_chunk_discard(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk) 682{ 683 bool committed; 684 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; 685 686 chunk_deregister(chunk, &chunk->node); 687 688 committed = (arena_mapbits_decommitted_get(chunk, map_bias) == 0); 689 if (!committed) { 690 /* 691 * Decommit the header. Mark the chunk as decommitted even if 692 * header decommit fails, since treating a partially committed 693 * chunk as committed has a high potential for causing later 694 * access of decommitted memory. 695 */ 696 chunk_hooks = chunk_hooks_get(tsdn, arena); 697 chunk_hooks.decommit(chunk, chunksize, 0, map_bias << LG_PAGE, 698 arena->ind); 699 } 700 701 chunk_dalloc_cache(tsdn, arena, &chunk_hooks, (void *)chunk, chunksize, 702 committed); 703 704 if (config_stats) { 705 arena->stats.mapped -= chunksize; 706 arena->stats.metadata_mapped -= (map_bias << LG_PAGE); 707 } 708} 709 710static void 711arena_spare_discard(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *spare) 712{ 713 714 assert(arena->spare != spare); 715 716 if (arena_mapbits_dirty_get(spare, map_bias) != 0) { 717 arena_run_dirty_remove(arena, spare, map_bias, 718 chunk_npages-map_bias); 719 } 720 721 arena_chunk_discard(tsdn, arena, spare); 722} 723 724static void 725arena_chunk_dalloc(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk) 726{ 727 arena_chunk_t *spare; 728 729 assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); 730 assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); 731 assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == 732 arena_maxrun); 733 assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == 734 arena_maxrun); 735 assert(arena_mapbits_dirty_get(chunk, map_bias) == 736 arena_mapbits_dirty_get(chunk, chunk_npages-1)); 737 assert(arena_mapbits_decommitted_get(chunk, map_bias) == 738 arena_mapbits_decommitted_get(chunk, chunk_npages-1)); 739 740 /* Remove run from runs_avail, so that the arena does not use it. */ 741 arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias); 742 743 ql_remove(&arena->achunks, &chunk->node, ql_link); 744 spare = arena->spare; 745 arena->spare = chunk; 746 if (spare != NULL) 747 arena_spare_discard(tsdn, arena, spare); 748} 749 750static void 751arena_huge_malloc_stats_update(arena_t *arena, size_t usize) 752{ 753 szind_t index = size2index(usize) - nlclasses - NBINS; 754 755 cassert(config_stats); 756 757 arena->stats.nmalloc_huge++; 758 arena->stats.allocated_huge += usize; 759 arena->stats.hstats[index].nmalloc++; 760 arena->stats.hstats[index].curhchunks++; 761} 762 763static void 764arena_huge_malloc_stats_update_undo(arena_t *arena, size_t usize) 765{ 766 szind_t index = size2index(usize) - nlclasses - NBINS; 767 768 cassert(config_stats); 769 770 arena->stats.nmalloc_huge--; 771 arena->stats.allocated_huge -= usize; 772 arena->stats.hstats[index].nmalloc--; 773 arena->stats.hstats[index].curhchunks--; 774} 775 776static void 777arena_huge_dalloc_stats_update(arena_t *arena, size_t usize) 778{ 779 szind_t index = size2index(usize) - nlclasses - NBINS; 780 781 cassert(config_stats); 782 783 arena->stats.ndalloc_huge++; 784 arena->stats.allocated_huge -= usize; 785 arena->stats.hstats[index].ndalloc++; 786 arena->stats.hstats[index].curhchunks--; 787} 788 789static void 790arena_huge_reset_stats_cancel(arena_t *arena, size_t usize) 791{ 792 szind_t index = size2index(usize) - nlclasses - NBINS; 793 794 cassert(config_stats); 795 796 arena->stats.ndalloc_huge++; 797 arena->stats.hstats[index].ndalloc--; 798} 799 800static void 801arena_huge_dalloc_stats_update_undo(arena_t *arena, size_t usize) 802{ 803 szind_t index = size2index(usize) - nlclasses - NBINS; 804 805 cassert(config_stats); 806 807 arena->stats.ndalloc_huge--; 808 arena->stats.allocated_huge += usize; 809 arena->stats.hstats[index].ndalloc--; 810 arena->stats.hstats[index].curhchunks++; 811} 812 813static void 814arena_huge_ralloc_stats_update(arena_t *arena, size_t oldsize, size_t usize) 815{ 816 817 arena_huge_dalloc_stats_update(arena, oldsize); 818 arena_huge_malloc_stats_update(arena, usize); 819} 820 821static void 822arena_huge_ralloc_stats_update_undo(arena_t *arena, size_t oldsize, 823 size_t usize) 824{ 825 826 arena_huge_dalloc_stats_update_undo(arena, oldsize); 827 arena_huge_malloc_stats_update_undo(arena, usize); 828} 829 830extent_node_t * 831arena_node_alloc(tsdn_t *tsdn, arena_t *arena) 832{ 833 extent_node_t *node; 834 835 malloc_mutex_lock(tsdn, &arena->node_cache_mtx); 836 node = ql_last(&arena->node_cache, ql_link); 837 if (node == NULL) { 838 malloc_mutex_unlock(tsdn, &arena->node_cache_mtx); 839 return (base_alloc(tsdn, sizeof(extent_node_t))); 840 } 841 ql_tail_remove(&arena->node_cache, extent_node_t, ql_link); 842 malloc_mutex_unlock(tsdn, &arena->node_cache_mtx); 843 return (node); 844} 845 846void 847arena_node_dalloc(tsdn_t *tsdn, arena_t *arena, extent_node_t *node) 848{ 849 850 malloc_mutex_lock(tsdn, &arena->node_cache_mtx); 851 ql_elm_new(node, ql_link); 852 ql_tail_insert(&arena->node_cache, node, ql_link); 853 malloc_mutex_unlock(tsdn, &arena->node_cache_mtx); 854} 855 856static void * 857arena_chunk_alloc_huge_hard(tsdn_t *tsdn, arena_t *arena, 858 chunk_hooks_t *chunk_hooks, size_t usize, size_t alignment, bool *zero, 859 size_t csize) 860{ 861 void *ret; 862 bool commit = true; 863 864 ret = chunk_alloc_wrapper(tsdn, arena, chunk_hooks, NULL, csize, 865 alignment, zero, &commit); 866 if (ret == NULL) { 867 /* Revert optimistic stats updates. */ 868 malloc_mutex_lock(tsdn, &arena->lock); 869 if (config_stats) { 870 arena_huge_malloc_stats_update_undo(arena, usize); 871 arena->stats.mapped -= usize; 872 } 873 arena_nactive_sub(arena, usize >> LG_PAGE); 874 malloc_mutex_unlock(tsdn, &arena->lock); 875 } 876 877 return (ret); 878} 879 880void * 881arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize, 882 size_t alignment, bool *zero) 883{ 884 void *ret; 885 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; 886 size_t csize = CHUNK_CEILING(usize); 887 bool commit = true; 888 889 malloc_mutex_lock(tsdn, &arena->lock); 890 891 /* Optimistically update stats. */ 892 if (config_stats) { 893 arena_huge_malloc_stats_update(arena, usize); 894 arena->stats.mapped += usize; 895 } 896 arena_nactive_add(arena, usize >> LG_PAGE); 897 898 ret = chunk_alloc_cache(tsdn, arena, &chunk_hooks, NULL, csize, 899 alignment, zero, &commit, true); 900 malloc_mutex_unlock(tsdn, &arena->lock); 901 if (ret == NULL) { 902 ret = arena_chunk_alloc_huge_hard(tsdn, arena, &chunk_hooks, 903 usize, alignment, zero, csize); 904 } 905 906 return (ret); 907} 908 909void 910arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, void *chunk, size_t usize) 911{ 912 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; 913 size_t csize; 914 915 csize = CHUNK_CEILING(usize); 916 malloc_mutex_lock(tsdn, &arena->lock); 917 if (config_stats) { 918 arena_huge_dalloc_stats_update(arena, usize); 919 arena->stats.mapped -= usize; 920 } 921 arena_nactive_sub(arena, usize >> LG_PAGE); 922 923 chunk_dalloc_cache(tsdn, arena, &chunk_hooks, chunk, csize, true); 924 malloc_mutex_unlock(tsdn, &arena->lock); 925} 926 927void 928arena_chunk_ralloc_huge_similar(tsdn_t *tsdn, arena_t *arena, void *chunk, 929 size_t oldsize, size_t usize) 930{ 931 932 assert(CHUNK_CEILING(oldsize) == CHUNK_CEILING(usize)); 933 assert(oldsize != usize); 934 935 malloc_mutex_lock(tsdn, &arena->lock); 936 if (config_stats) 937 arena_huge_ralloc_stats_update(arena, oldsize, usize); 938 if (oldsize < usize) 939 arena_nactive_add(arena, (usize - oldsize) >> LG_PAGE); 940 else 941 arena_nactive_sub(arena, (oldsize - usize) >> LG_PAGE); 942 malloc_mutex_unlock(tsdn, &arena->lock); 943} 944 945void 946arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena, void *chunk, 947 size_t oldsize, size_t usize) 948{ 949 size_t udiff = oldsize - usize; 950 size_t cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize); 951 952 malloc_mutex_lock(tsdn, &arena->lock); 953 if (config_stats) { 954 arena_huge_ralloc_stats_update(arena, oldsize, usize); 955 if (cdiff != 0) 956 arena->stats.mapped -= cdiff; 957 } 958 arena_nactive_sub(arena, udiff >> LG_PAGE); 959 960 if (cdiff != 0) { 961 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; 962 void *nchunk = (void *)((uintptr_t)chunk + 963 CHUNK_CEILING(usize)); 964 965 chunk_dalloc_cache(tsdn, arena, &chunk_hooks, nchunk, cdiff, 966 true); 967 } 968 malloc_mutex_unlock(tsdn, &arena->lock); 969} 970 971static bool 972arena_chunk_ralloc_huge_expand_hard(tsdn_t *tsdn, arena_t *arena, 973 chunk_hooks_t *chunk_hooks, void *chunk, size_t oldsize, size_t usize, 974 bool *zero, void *nchunk, size_t udiff, size_t cdiff) 975{ 976 bool err; 977 bool commit = true; 978 979 err = (chunk_alloc_wrapper(tsdn, arena, chunk_hooks, nchunk, cdiff, 980 chunksize, zero, &commit) == NULL); 981 if (err) { 982 /* Revert optimistic stats updates. */ 983 malloc_mutex_lock(tsdn, &arena->lock); 984 if (config_stats) { 985 arena_huge_ralloc_stats_update_undo(arena, oldsize, 986 usize); 987 arena->stats.mapped -= cdiff; 988 } 989 arena_nactive_sub(arena, udiff >> LG_PAGE); 990 malloc_mutex_unlock(tsdn, &arena->lock); 991 } else if (chunk_hooks->merge(chunk, CHUNK_CEILING(oldsize), nchunk, 992 cdiff, true, arena->ind)) { 993 chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, nchunk, cdiff, 994 *zero, true); 995 err = true; 996 } 997 return (err); 998} 999 1000bool 1001arena_chunk_ralloc_huge_expand(tsdn_t *tsdn, arena_t *arena, void *chunk, 1002 size_t oldsize, size_t usize, bool *zero) 1003{ 1004 bool err; 1005 chunk_hooks_t chunk_hooks = chunk_hooks_get(tsdn, arena); 1006 void *nchunk = (void *)((uintptr_t)chunk + CHUNK_CEILING(oldsize)); 1007 size_t udiff = usize - oldsize; 1008 size_t cdiff = CHUNK_CEILING(usize) - CHUNK_CEILING(oldsize); 1009 bool commit = true; 1010 1011 malloc_mutex_lock(tsdn, &arena->lock); 1012 1013 /* Optimistically update stats. */ 1014 if (config_stats) { 1015 arena_huge_ralloc_stats_update(arena, oldsize, usize); 1016 arena->stats.mapped += cdiff; 1017 } 1018 arena_nactive_add(arena, udiff >> LG_PAGE); 1019 1020 err = (chunk_alloc_cache(tsdn, arena, &chunk_hooks, nchunk, cdiff, 1021 chunksize, zero, &commit, true) == NULL); 1022 malloc_mutex_unlock(tsdn, &arena->lock); 1023 if (err) { 1024 err = arena_chunk_ralloc_huge_expand_hard(tsdn, arena, 1025 &chunk_hooks, chunk, oldsize, usize, zero, nchunk, udiff, 1026 cdiff); 1027 } else if (chunk_hooks.merge(chunk, CHUNK_CEILING(oldsize), nchunk, 1028 cdiff, true, arena->ind)) { 1029 chunk_dalloc_wrapper(tsdn, arena, &chunk_hooks, nchunk, cdiff, 1030 *zero, true); 1031 err = true; 1032 } 1033 1034 return (err); 1035} 1036 1037/* 1038 * Do first-best-fit run selection, i.e. select the lowest run that best fits. 1039 * Run sizes are indexed, so not all candidate runs are necessarily exactly the 1040 * same size. 1041 */ 1042static arena_run_t * 1043arena_run_first_best_fit(arena_t *arena, size_t size) 1044{ 1045 pszind_t pind, i; 1046 1047 pind = psz2ind(run_quantize_ceil(size)); 1048 1049 for (i = pind; pind2sz(i) <= large_maxclass; i++) { 1050 arena_chunk_map_misc_t *miscelm = arena_run_heap_first( 1051 &arena->runs_avail[i]); 1052 if (miscelm != NULL) 1053 return (&miscelm->run); 1054 } 1055 1056 return (NULL); 1057} 1058 1059static arena_run_t * 1060arena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero) 1061{ 1062 arena_run_t *run = arena_run_first_best_fit(arena, size); 1063 if (run != NULL) { 1064 if (arena_run_split_large(arena, run, size, zero)) 1065 run = NULL; 1066 } 1067 return (run); 1068} 1069 1070static arena_run_t * 1071arena_run_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t size, bool zero) 1072{ 1073 arena_chunk_t *chunk; 1074 arena_run_t *run; 1075 1076 assert(size <= arena_maxrun); 1077 assert(size == PAGE_CEILING(size)); 1078 1079 /* Search the arena's chunks for the lowest best fit. */ 1080 run = arena_run_alloc_large_helper(arena, size, zero); 1081 if (run != NULL) 1082 return (run); 1083 1084 /* 1085 * No usable runs. Create a new chunk from which to allocate the run. 1086 */ 1087 chunk = arena_chunk_alloc(tsdn, arena); 1088 if (chunk != NULL) { 1089 run = &arena_miscelm_get_mutable(chunk, map_bias)->run; 1090 if (arena_run_split_large(arena, run, size, zero)) 1091 run = NULL; 1092 return (run); 1093 } 1094 1095 /* 1096 * arena_chunk_alloc() failed, but another thread may have made 1097 * sufficient memory available while this one dropped arena->lock in 1098 * arena_chunk_alloc(), so search one more time. 1099 */ 1100 return (arena_run_alloc_large_helper(arena, size, zero)); 1101} 1102 1103static arena_run_t * 1104arena_run_alloc_small_helper(arena_t *arena, size_t size, szind_t binind) 1105{ 1106 arena_run_t *run = arena_run_first_best_fit(arena, size); 1107 if (run != NULL) { 1108 if (arena_run_split_small(arena, run, size, binind)) 1109 run = NULL; 1110 } 1111 return (run); 1112} 1113 1114static arena_run_t * 1115arena_run_alloc_small(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t binind) 1116{ 1117 arena_chunk_t *chunk; 1118 arena_run_t *run; 1119 1120 assert(size <= arena_maxrun); 1121 assert(size == PAGE_CEILING(size)); 1122 assert(binind != BININD_INVALID); 1123 1124 /* Search the arena's chunks for the lowest best fit. */ 1125 run = arena_run_alloc_small_helper(arena, size, binind); 1126 if (run != NULL) 1127 return (run); 1128 1129 /* 1130 * No usable runs. Create a new chunk from which to allocate the run. 1131 */ 1132 chunk = arena_chunk_alloc(tsdn, arena); 1133 if (chunk != NULL) { 1134 run = &arena_miscelm_get_mutable(chunk, map_bias)->run; 1135 if (arena_run_split_small(arena, run, size, binind)) 1136 run = NULL; 1137 return (run); 1138 } 1139 1140 /* 1141 * arena_chunk_alloc() failed, but another thread may have made 1142 * sufficient memory available while this one dropped arena->lock in 1143 * arena_chunk_alloc(), so search one more time. 1144 */ 1145 return (arena_run_alloc_small_helper(arena, size, binind)); 1146} 1147 1148static bool 1149arena_lg_dirty_mult_valid(ssize_t lg_dirty_mult) 1150{ 1151 1152 return (lg_dirty_mult >= -1 && lg_dirty_mult < (ssize_t)(sizeof(size_t) 1153 << 3)); 1154} 1155 1156ssize_t 1157arena_lg_dirty_mult_get(tsdn_t *tsdn, arena_t *arena) 1158{ 1159 ssize_t lg_dirty_mult; 1160 1161 malloc_mutex_lock(tsdn, &arena->lock); 1162 lg_dirty_mult = arena->lg_dirty_mult; 1163 malloc_mutex_unlock(tsdn, &arena->lock); 1164 1165 return (lg_dirty_mult); 1166} 1167 1168bool 1169arena_lg_dirty_mult_set(tsdn_t *tsdn, arena_t *arena, ssize_t lg_dirty_mult) 1170{ 1171 1172 if (!arena_lg_dirty_mult_valid(lg_dirty_mult)) 1173 return (true); 1174 1175 malloc_mutex_lock(tsdn, &arena->lock); 1176 arena->lg_dirty_mult = lg_dirty_mult; 1177 arena_maybe_purge(tsdn, arena); 1178 malloc_mutex_unlock(tsdn, &arena->lock); 1179 1180 return (false); 1181} 1182 1183static void 1184arena_decay_deadline_init(arena_t *arena) 1185{ 1186 1187 assert(opt_purge == purge_mode_decay); 1188 1189 /* 1190 * Generate a new deadline that is uniformly random within the next 1191 * epoch after the current one. 1192 */ 1193 nstime_copy(&arena->decay.deadline, &arena->decay.epoch); 1194 nstime_add(&arena->decay.deadline, &arena->decay.interval); 1195 if (arena->decay.time > 0) { 1196 nstime_t jitter; 1197 1198 nstime_init(&jitter, prng_range(&arena->decay.jitter_state, 1199 nstime_ns(&arena->decay.interval))); 1200 nstime_add(&arena->decay.deadline, &jitter); 1201 } 1202} 1203 1204static bool 1205arena_decay_deadline_reached(const arena_t *arena, const nstime_t *time) 1206{ 1207 1208 assert(opt_purge == purge_mode_decay); 1209 1210 return (nstime_compare(&arena->decay.deadline, time) <= 0); 1211} 1212 1213static size_t 1214arena_decay_backlog_npages_limit(const arena_t *arena) 1215{ 1216 static const uint64_t h_steps[] = { 1217#define STEP(step, h, x, y) \ 1218 h, 1219 SMOOTHSTEP 1220#undef STEP 1221 }; 1222 uint64_t sum; 1223 size_t npages_limit_backlog; 1224 unsigned i; 1225 1226 assert(opt_purge == purge_mode_decay); 1227 1228 /* 1229 * For each element of decay_backlog, multiply by the corresponding 1230 * fixed-point smoothstep decay factor. Sum the products, then divide 1231 * to round down to the nearest whole number of pages. 1232 */ 1233 sum = 0; 1234 for (i = 0; i < SMOOTHSTEP_NSTEPS; i++) 1235 sum += arena->decay.backlog[i] * h_steps[i]; 1236 npages_limit_backlog = (size_t)(sum >> SMOOTHSTEP_BFP); 1237 1238 return (npages_limit_backlog); 1239} 1240 1241static void 1242arena_decay_backlog_update_last(arena_t *arena) 1243{ 1244 size_t ndirty_delta = (arena->ndirty > arena->decay.ndirty) ? 1245 arena->ndirty - arena->decay.ndirty : 0; 1246 arena->decay.backlog[SMOOTHSTEP_NSTEPS-1] = ndirty_delta; 1247} 1248 1249static void 1250arena_decay_backlog_update(arena_t *arena, uint64_t nadvance_u64) 1251{ 1252 1253 if (nadvance_u64 >= SMOOTHSTEP_NSTEPS) { 1254 memset(arena->decay.backlog, 0, (SMOOTHSTEP_NSTEPS-1) * 1255 sizeof(size_t)); 1256 } else { 1257 size_t nadvance_z = (size_t)nadvance_u64; 1258 1259 assert((uint64_t)nadvance_z == nadvance_u64); 1260 1261 memmove(arena->decay.backlog, &arena->decay.backlog[nadvance_z], 1262 (SMOOTHSTEP_NSTEPS - nadvance_z) * sizeof(size_t)); 1263 if (nadvance_z > 1) { 1264 memset(&arena->decay.backlog[SMOOTHSTEP_NSTEPS - 1265 nadvance_z], 0, (nadvance_z-1) * sizeof(size_t)); 1266 } 1267 } 1268 1269 arena_decay_backlog_update_last(arena); 1270} 1271 1272static void 1273arena_decay_epoch_advance_helper(arena_t *arena, const nstime_t *time) 1274{ 1275 uint64_t nadvance_u64; 1276 nstime_t delta; 1277 1278 assert(opt_purge == purge_mode_decay); 1279 assert(arena_decay_deadline_reached(arena, time)); 1280 1281 nstime_copy(&delta, time); 1282 nstime_subtract(&delta, &arena->decay.epoch); 1283 nadvance_u64 = nstime_divide(&delta, &arena->decay.interval); 1284 assert(nadvance_u64 > 0); 1285 1286 /* Add nadvance_u64 decay intervals to epoch. */ 1287 nstime_copy(&delta, &arena->decay.interval); 1288 nstime_imultiply(&delta, nadvance_u64); 1289 nstime_add(&arena->decay.epoch, &delta); 1290 1291 /* Set a new deadline. */ 1292 arena_decay_deadline_init(arena); 1293 1294 /* Update the backlog. */ 1295 arena_decay_backlog_update(arena, nadvance_u64); 1296} 1297 1298static void 1299arena_decay_epoch_advance_purge(tsdn_t *tsdn, arena_t *arena) 1300{ 1301 size_t ndirty_limit = arena_decay_backlog_npages_limit(arena); 1302 1303 if (arena->ndirty > ndirty_limit) 1304 arena_purge_to_limit(tsdn, arena, ndirty_limit); 1305 arena->decay.ndirty = arena->ndirty; 1306} 1307 1308static void 1309arena_decay_epoch_advance(tsdn_t *tsdn, arena_t *arena, const nstime_t *time) 1310{ 1311 1312 arena_decay_epoch_advance_helper(arena, time); 1313 arena_decay_epoch_advance_purge(tsdn, arena); 1314} 1315 1316static void 1317arena_decay_init(arena_t *arena, ssize_t decay_time) 1318{ 1319 1320 arena->decay.time = decay_time; 1321 if (decay_time > 0) { 1322 nstime_init2(&arena->decay.interval, decay_time, 0); 1323 nstime_idivide(&arena->decay.interval, SMOOTHSTEP_NSTEPS); 1324 } 1325 1326 nstime_init(&arena->decay.epoch, 0); 1327 nstime_update(&arena->decay.epoch); 1328 arena->decay.jitter_state = (uint64_t)(uintptr_t)arena; 1329 arena_decay_deadline_init(arena); 1330 arena->decay.ndirty = arena->ndirty; 1331 memset(arena->decay.backlog, 0, SMOOTHSTEP_NSTEPS * sizeof(size_t)); 1332} 1333 1334static bool 1335arena_decay_time_valid(ssize_t decay_time) 1336{ 1337 1338 if (decay_time < -1) 1339 return (false); 1340 if (decay_time == -1 || (uint64_t)decay_time <= NSTIME_SEC_MAX) 1341 return (true); 1342 return (false); 1343} 1344 1345ssize_t 1346arena_decay_time_get(tsdn_t *tsdn, arena_t *arena) 1347{ 1348 ssize_t decay_time; 1349 1350 malloc_mutex_lock(tsdn, &arena->lock); 1351 decay_time = arena->decay.time; 1352 malloc_mutex_unlock(tsdn, &arena->lock); 1353 1354 return (decay_time); 1355} 1356 1357bool 1358arena_decay_time_set(tsdn_t *tsdn, arena_t *arena, ssize_t decay_time) 1359{ 1360 1361 if (!arena_decay_time_valid(decay_time)) 1362 return (true); 1363 1364 malloc_mutex_lock(tsdn, &arena->lock); 1365 /* 1366 * Restart decay backlog from scratch, which may cause many dirty pages 1367 * to be immediately purged. It would conceptually be possible to map 1368 * the old backlog onto the new backlog, but there is no justification 1369 * for such complexity since decay_time changes are intended to be 1370 * infrequent, either between the {-1, 0, >0} states, or a one-time 1371 * arbitrary change during initial arena configuration. 1372 */ 1373 arena_decay_init(arena, decay_time); 1374 arena_maybe_purge(tsdn, arena); 1375 malloc_mutex_unlock(tsdn, &arena->lock); 1376 1377 return (false); 1378} 1379 1380static void 1381arena_maybe_purge_ratio(tsdn_t *tsdn, arena_t *arena) 1382{ 1383 1384 assert(opt_purge == purge_mode_ratio); 1385 1386 /* Don't purge if the option is disabled. */ 1387 if (arena->lg_dirty_mult < 0) 1388 return; 1389 1390 /* 1391 * Iterate, since preventing recursive purging could otherwise leave too 1392 * many dirty pages. 1393 */ 1394 while (true) { 1395 size_t threshold = (arena->nactive >> arena->lg_dirty_mult); 1396 if (threshold < chunk_npages) 1397 threshold = chunk_npages; 1398 /* 1399 * Don't purge unless the number of purgeable pages exceeds the 1400 * threshold. 1401 */ 1402 if (arena->ndirty <= threshold) 1403 return; 1404 arena_purge_to_limit(tsdn, arena, threshold); 1405 } 1406} 1407 1408static void 1409arena_maybe_purge_decay(tsdn_t *tsdn, arena_t *arena) 1410{ 1411 nstime_t time; 1412 1413 assert(opt_purge == purge_mode_decay); 1414 1415 /* Purge all or nothing if the option is disabled. */ 1416 if (arena->decay.time <= 0) { 1417 if (arena->decay.time == 0) 1418 arena_purge_to_limit(tsdn, arena, 0); 1419 return; 1420 } 1421 1422 nstime_init(&time, 0); 1423 nstime_update(&time); 1424 if (unlikely(!nstime_monotonic() && nstime_compare(&arena->decay.epoch, 1425 &time) > 0)) { 1426 /* 1427 * Time went backwards. Move the epoch back in time and 1428 * generate a new deadline, with the expectation that time 1429 * typically flows forward for long enough periods of time that 1430 * epochs complete. Unfortunately, this strategy is susceptible 1431 * to clock jitter triggering premature epoch advances, but 1432 * clock jitter estimation and compensation isn't feasible here 1433 * because calls into this code are event-driven. 1434 */ 1435 nstime_copy(&arena->decay.epoch, &time); 1436 arena_decay_deadline_init(arena); 1437 } else { 1438 /* Verify that time does not go backwards. */ 1439 assert(nstime_compare(&arena->decay.epoch, &time) <= 0); 1440 } 1441 1442 /* 1443 * If the deadline has been reached, advance to the current epoch and 1444 * purge to the new limit if necessary. Note that dirty pages created 1445 * during the current epoch are not subject to purge until a future 1446 * epoch, so as a result purging only happens during epoch advances. 1447 */ 1448 if (arena_decay_deadline_reached(arena, &time)) 1449 arena_decay_epoch_advance(tsdn, arena, &time); 1450} 1451 1452void 1453arena_maybe_purge(tsdn_t *tsdn, arena_t *arena) 1454{ 1455 1456 /* Don't recursively purge. */ 1457 if (arena->purging) 1458 return; 1459 1460 if (opt_purge == purge_mode_ratio) 1461 arena_maybe_purge_ratio(tsdn, arena); 1462 else 1463 arena_maybe_purge_decay(tsdn, arena); 1464} 1465 1466static size_t 1467arena_dirty_count(arena_t *arena) 1468{ 1469 size_t ndirty = 0; 1470 arena_runs_dirty_link_t *rdelm; 1471 extent_node_t *chunkselm; 1472 1473 for (rdelm = qr_next(&arena->runs_dirty, rd_link), 1474 chunkselm = qr_next(&arena->chunks_cache, cc_link); 1475 rdelm != &arena->runs_dirty; rdelm = qr_next(rdelm, rd_link)) { 1476 size_t npages; 1477 1478 if (rdelm == &chunkselm->rd) { 1479 npages = extent_node_size_get(chunkselm) >> LG_PAGE; 1480 chunkselm = qr_next(chunkselm, cc_link); 1481 } else { 1482 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( 1483 rdelm); 1484 arena_chunk_map_misc_t *miscelm = 1485 arena_rd_to_miscelm(rdelm); 1486 size_t pageind = arena_miscelm_to_pageind(miscelm); 1487 assert(arena_mapbits_allocated_get(chunk, pageind) == 1488 0); 1489 assert(arena_mapbits_large_get(chunk, pageind) == 0); 1490 assert(arena_mapbits_dirty_get(chunk, pageind) != 0); 1491 npages = arena_mapbits_unallocated_size_get(chunk, 1492 pageind) >> LG_PAGE; 1493 } 1494 ndirty += npages; 1495 } 1496 1497 return (ndirty); 1498} 1499 1500static size_t 1501arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, 1502 size_t ndirty_limit, arena_runs_dirty_link_t *purge_runs_sentinel, 1503 extent_node_t *purge_chunks_sentinel) 1504{ 1505 arena_runs_dirty_link_t *rdelm, *rdelm_next; 1506 extent_node_t *chunkselm; 1507 size_t nstashed = 0; 1508 1509 /* Stash runs/chunks according to ndirty_limit. */ 1510 for (rdelm = qr_next(&arena->runs_dirty, rd_link), 1511 chunkselm = qr_next(&arena->chunks_cache, cc_link); 1512 rdelm != &arena->runs_dirty; rdelm = rdelm_next) { 1513 size_t npages; 1514 rdelm_next = qr_next(rdelm, rd_link); 1515 1516 if (rdelm == &chunkselm->rd) { 1517 extent_node_t *chunkselm_next; 1518 bool zero, commit; 1519 UNUSED void *chunk; 1520 1521 npages = extent_node_size_get(chunkselm) >> LG_PAGE; 1522 if (opt_purge == purge_mode_decay && arena->ndirty - 1523 (nstashed + npages) < ndirty_limit) 1524 break; 1525 1526 chunkselm_next = qr_next(chunkselm, cc_link); 1527 /* 1528 * Allocate. chunkselm remains valid due to the 1529 * dalloc_node=false argument to chunk_alloc_cache(). 1530 */ 1531 zero = false; 1532 commit = false; 1533 chunk = chunk_alloc_cache(tsdn, arena, chunk_hooks, 1534 extent_node_addr_get(chunkselm), 1535 extent_node_size_get(chunkselm), chunksize, &zero, 1536 &commit, false); 1537 assert(chunk == extent_node_addr_get(chunkselm)); 1538 assert(zero == extent_node_zeroed_get(chunkselm)); 1539 extent_node_dirty_insert(chunkselm, purge_runs_sentinel, 1540 purge_chunks_sentinel); 1541 assert(npages == (extent_node_size_get(chunkselm) >> 1542 LG_PAGE)); 1543 chunkselm = chunkselm_next; 1544 } else { 1545 arena_chunk_t *chunk = 1546 (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm); 1547 arena_chunk_map_misc_t *miscelm = 1548 arena_rd_to_miscelm(rdelm); 1549 size_t pageind = arena_miscelm_to_pageind(miscelm); 1550 arena_run_t *run = &miscelm->run; 1551 size_t run_size = 1552 arena_mapbits_unallocated_size_get(chunk, pageind); 1553 1554 npages = run_size >> LG_PAGE; 1555 if (opt_purge == purge_mode_decay && arena->ndirty - 1556 (nstashed + npages) < ndirty_limit) 1557 break; 1558 1559 assert(pageind + npages <= chunk_npages); 1560 assert(arena_mapbits_dirty_get(chunk, pageind) == 1561 arena_mapbits_dirty_get(chunk, pageind+npages-1)); 1562 1563 /* 1564 * If purging the spare chunk's run, make it available 1565 * prior to allocation. 1566 */ 1567 if (chunk == arena->spare) 1568 arena_chunk_alloc(tsdn, arena); 1569 1570 /* Temporarily allocate the free dirty run. */ 1571 arena_run_split_large(arena, run, run_size, false); 1572 /* Stash. */ 1573 if (false) 1574 qr_new(rdelm, rd_link); /* Redundant. */ 1575 else { 1576 assert(qr_next(rdelm, rd_link) == rdelm); 1577 assert(qr_prev(rdelm, rd_link) == rdelm); 1578 } 1579 qr_meld(purge_runs_sentinel, rdelm, rd_link); 1580 } 1581 1582 nstashed += npages; 1583 if (opt_purge == purge_mode_ratio && arena->ndirty - nstashed <= 1584 ndirty_limit) 1585 break; 1586 } 1587 1588 return (nstashed); 1589} 1590 1591static size_t 1592arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, 1593 arena_runs_dirty_link_t *purge_runs_sentinel, 1594 extent_node_t *purge_chunks_sentinel) 1595{ 1596 size_t npurged, nmadvise; 1597 arena_runs_dirty_link_t *rdelm; 1598 extent_node_t *chunkselm; 1599 1600 if (config_stats) 1601 nmadvise = 0; 1602 npurged = 0; 1603 1604 malloc_mutex_unlock(tsdn, &arena->lock); 1605 for (rdelm = qr_next(purge_runs_sentinel, rd_link), 1606 chunkselm = qr_next(purge_chunks_sentinel, cc_link); 1607 rdelm != purge_runs_sentinel; rdelm = qr_next(rdelm, rd_link)) { 1608 size_t npages; 1609 1610 if (rdelm == &chunkselm->rd) { 1611 /* 1612 * Don't actually purge the chunk here because 1) 1613 * chunkselm is embedded in the chunk and must remain 1614 * valid, and 2) we deallocate the chunk in 1615 * arena_unstash_purged(), where it is destroyed, 1616 * decommitted, or purged, depending on chunk 1617 * deallocation policy. 1618 */ 1619 size_t size = extent_node_size_get(chunkselm); 1620 npages = size >> LG_PAGE; 1621 chunkselm = qr_next(chunkselm, cc_link); 1622 } else { 1623 size_t pageind, run_size, flag_unzeroed, flags, i; 1624 bool decommitted; 1625 arena_chunk_t *chunk = 1626 (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm); 1627 arena_chunk_map_misc_t *miscelm = 1628 arena_rd_to_miscelm(rdelm); 1629 pageind = arena_miscelm_to_pageind(miscelm); 1630 run_size = arena_mapbits_large_size_get(chunk, pageind); 1631 npages = run_size >> LG_PAGE; 1632 1633 assert(pageind + npages <= chunk_npages); 1634 assert(!arena_mapbits_decommitted_get(chunk, pageind)); 1635 assert(!arena_mapbits_decommitted_get(chunk, 1636 pageind+npages-1)); 1637 decommitted = !chunk_hooks->decommit(chunk, chunksize, 1638 pageind << LG_PAGE, npages << LG_PAGE, arena->ind); 1639 if (decommitted) { 1640 flag_unzeroed = 0; 1641 flags = CHUNK_MAP_DECOMMITTED; 1642 } else { 1643 flag_unzeroed = chunk_purge_wrapper(tsdn, arena, 1644 chunk_hooks, chunk, chunksize, pageind << 1645 LG_PAGE, run_size) ? CHUNK_MAP_UNZEROED : 0; 1646 flags = flag_unzeroed; 1647 } 1648 arena_mapbits_large_set(chunk, pageind+npages-1, 0, 1649 flags); 1650 arena_mapbits_large_set(chunk, pageind, run_size, 1651 flags); 1652 1653 /* 1654 * Set the unzeroed flag for internal pages, now that 1655 * chunk_purge_wrapper() has returned whether the pages 1656 * were zeroed as a side effect of purging. This chunk 1657 * map modification is safe even though the arena mutex 1658 * isn't currently owned by this thread, because the run 1659 * is marked as allocated, thus protecting it from being 1660 * modified by any other thread. As long as these 1661 * writes don't perturb the first and last elements' 1662 * CHUNK_MAP_ALLOCATED bits, behavior is well defined. 1663 */ 1664 for (i = 1; i < npages-1; i++) { 1665 arena_mapbits_internal_set(chunk, pageind+i, 1666 flag_unzeroed); 1667 } 1668 } 1669 1670 npurged += npages; 1671 if (config_stats) 1672 nmadvise++; 1673 } 1674 malloc_mutex_lock(tsdn, &arena->lock); 1675 1676 if (config_stats) { 1677 arena->stats.nmadvise += nmadvise; 1678 arena->stats.purged += npurged; 1679 } 1680 1681 return (npurged); 1682} 1683 1684static void 1685arena_unstash_purged(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, 1686 arena_runs_dirty_link_t *purge_runs_sentinel, 1687 extent_node_t *purge_chunks_sentinel) 1688{ 1689 arena_runs_dirty_link_t *rdelm, *rdelm_next; 1690 extent_node_t *chunkselm; 1691 1692 /* Deallocate chunks/runs. */ 1693 for (rdelm = qr_next(purge_runs_sentinel, rd_link), 1694 chunkselm = qr_next(purge_chunks_sentinel, cc_link); 1695 rdelm != purge_runs_sentinel; rdelm = rdelm_next) { 1696 rdelm_next = qr_next(rdelm, rd_link); 1697 if (rdelm == &chunkselm->rd) { 1698 extent_node_t *chunkselm_next = qr_next(chunkselm, 1699 cc_link); 1700 void *addr = extent_node_addr_get(chunkselm); 1701 size_t size = extent_node_size_get(chunkselm); 1702 bool zeroed = extent_node_zeroed_get(chunkselm); 1703 bool committed = extent_node_committed_get(chunkselm); 1704 extent_node_dirty_remove(chunkselm); 1705 arena_node_dalloc(tsdn, arena, chunkselm); 1706 chunkselm = chunkselm_next; 1707 chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, addr, 1708 size, zeroed, committed); 1709 } else { 1710 arena_chunk_t *chunk = 1711 (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm); 1712 arena_chunk_map_misc_t *miscelm = 1713 arena_rd_to_miscelm(rdelm); 1714 size_t pageind = arena_miscelm_to_pageind(miscelm); 1715 bool decommitted = (arena_mapbits_decommitted_get(chunk, 1716 pageind) != 0); 1717 arena_run_t *run = &miscelm->run; 1718 qr_remove(rdelm, rd_link); 1719 arena_run_dalloc(tsdn, arena, run, false, true, 1720 decommitted); 1721 } 1722 } 1723} 1724 1725/* 1726 * NB: ndirty_limit is interpreted differently depending on opt_purge: 1727 * - purge_mode_ratio: Purge as few dirty run/chunks as possible to reach the 1728 * desired state: 1729 * (arena->ndirty <= ndirty_limit) 1730 * - purge_mode_decay: Purge as many dirty runs/chunks as possible without 1731 * violating the invariant: 1732 * (arena->ndirty >= ndirty_limit) 1733 */ 1734static void 1735arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena, size_t ndirty_limit) 1736{ 1737 chunk_hooks_t chunk_hooks = chunk_hooks_get(tsdn, arena); 1738 size_t npurge, npurged; 1739 arena_runs_dirty_link_t purge_runs_sentinel; 1740 extent_node_t purge_chunks_sentinel; 1741 1742 arena->purging = true; 1743 1744 /* 1745 * Calls to arena_dirty_count() are disabled even for debug builds 1746 * because overhead grows nonlinearly as memory usage increases. 1747 */ 1748 if (false && config_debug) { 1749 size_t ndirty = arena_dirty_count(arena); 1750 assert(ndirty == arena->ndirty); 1751 } 1752 assert(opt_purge != purge_mode_ratio || (arena->nactive >> 1753 arena->lg_dirty_mult) < arena->ndirty || ndirty_limit == 0); 1754 1755 qr_new(&purge_runs_sentinel, rd_link); 1756 extent_node_dirty_linkage_init(&purge_chunks_sentinel); 1757 1758 npurge = arena_stash_dirty(tsdn, arena, &chunk_hooks, ndirty_limit, 1759 &purge_runs_sentinel, &purge_chunks_sentinel); 1760 if (npurge == 0) 1761 goto label_return; 1762 npurged = arena_purge_stashed(tsdn, arena, &chunk_hooks, 1763 &purge_runs_sentinel, &purge_chunks_sentinel); 1764 assert(npurged == npurge); 1765 arena_unstash_purged(tsdn, arena, &chunk_hooks, &purge_runs_sentinel, 1766 &purge_chunks_sentinel); 1767 1768 if (config_stats) 1769 arena->stats.npurge++; 1770 1771label_return: 1772 arena->purging = false; 1773} 1774 1775void 1776arena_purge(tsdn_t *tsdn, arena_t *arena, bool all) 1777{ 1778 1779 malloc_mutex_lock(tsdn, &arena->lock); 1780 if (all) 1781 arena_purge_to_limit(tsdn, arena, 0); 1782 else 1783 arena_maybe_purge(tsdn, arena); 1784 malloc_mutex_unlock(tsdn, &arena->lock); 1785} 1786 1787static void 1788arena_achunk_prof_reset(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk) 1789{ 1790 size_t pageind, npages; 1791 1792 cassert(config_prof); 1793 assert(opt_prof); 1794 1795 /* 1796 * Iterate over the allocated runs and remove profiled allocations from 1797 * the sample set. 1798 */ 1799 for (pageind = map_bias; pageind < chunk_npages; pageind += npages) { 1800 if (arena_mapbits_allocated_get(chunk, pageind) != 0) { 1801 if (arena_mapbits_large_get(chunk, pageind) != 0) { 1802 void *ptr = (void *)((uintptr_t)chunk + (pageind 1803 << LG_PAGE)); 1804 size_t usize = isalloc(tsd_tsdn(tsd), ptr, 1805 config_prof); 1806 1807 prof_free(tsd, ptr, usize); 1808 npages = arena_mapbits_large_size_get(chunk, 1809 pageind) >> LG_PAGE; 1810 } else { 1811 /* Skip small run. */ 1812 size_t binind = arena_mapbits_binind_get(chunk, 1813 pageind); 1814 arena_bin_info_t *bin_info = 1815 &arena_bin_info[binind]; 1816 npages = bin_info->run_size >> LG_PAGE; 1817 } 1818 } else { 1819 /* Skip unallocated run. */ 1820 npages = arena_mapbits_unallocated_size_get(chunk, 1821 pageind) >> LG_PAGE; 1822 } 1823 assert(pageind + npages <= chunk_npages); 1824 } 1825} 1826 1827void 1828arena_reset(tsd_t *tsd, arena_t *arena) 1829{ 1830 unsigned i; 1831 extent_node_t *node; 1832 1833 /* 1834 * Locking in this function is unintuitive. The caller guarantees that 1835 * no concurrent operations are happening in this arena, but there are 1836 * still reasons that some locking is necessary: 1837 * 1838 * - Some of the functions in the transitive closure of calls assume 1839 * appropriate locks are held, and in some cases these locks are 1840 * temporarily dropped to avoid lock order reversal or deadlock due to 1841 * reentry. 1842 * - mallctl("epoch", ...) may concurrently refresh stats. While 1843 * strictly speaking this is a "concurrent operation", disallowing 1844 * stats refreshes would impose an inconvenient burden. 1845 */ 1846 1847 /* Remove large allocations from prof sample set. */ 1848 if (config_prof && opt_prof) { 1849 ql_foreach(node, &arena->achunks, ql_link) { 1850 arena_achunk_prof_reset(tsd, arena, 1851 extent_node_addr_get(node)); 1852 } 1853 } 1854 1855 /* Reset curruns for large size classes. */ 1856 if (config_stats) { 1857 for (i = 0; i < nlclasses; i++) 1858 arena->stats.lstats[i].curruns = 0; 1859 } 1860 1861 /* Huge allocations. */ 1862 malloc_mutex_lock(tsd_tsdn(tsd), &arena->huge_mtx); 1863 for (node = ql_last(&arena->huge, ql_link); node != NULL; node = 1864 ql_last(&arena->huge, ql_link)) { 1865 void *ptr = extent_node_addr_get(node); 1866 size_t usize; 1867 1868 malloc_mutex_unlock(tsd_tsdn(tsd), &arena->huge_mtx); 1869 if (config_stats || (config_prof && opt_prof)) 1870 usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); 1871 /* Remove huge allocation from prof sample set. */ 1872 if (config_prof && opt_prof) 1873 prof_free(tsd, ptr, usize); 1874 huge_dalloc(tsd_tsdn(tsd), ptr); 1875 malloc_mutex_lock(tsd_tsdn(tsd), &arena->huge_mtx); 1876 /* Cancel out unwanted effects on stats. */ 1877 if (config_stats) 1878 arena_huge_reset_stats_cancel(arena, usize); 1879 } 1880 malloc_mutex_unlock(tsd_tsdn(tsd), &arena->huge_mtx); 1881 1882 malloc_mutex_lock(tsd_tsdn(tsd), &arena->lock); 1883 1884 /* Bins. */ 1885 for (i = 0; i < NBINS; i++) { 1886 arena_bin_t *bin = &arena->bins[i]; 1887 malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); 1888 bin->runcur = NULL; 1889 arena_run_heap_new(&bin->runs); 1890 if (config_stats) { 1891 bin->stats.curregs = 0; 1892 bin->stats.curruns = 0; 1893 } 1894 malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); 1895 } 1896 1897 /* 1898 * Re-initialize runs_dirty such that the chunks_cache and runs_dirty 1899 * chains directly correspond. 1900 */ 1901 qr_new(&arena->runs_dirty, rd_link); 1902 for (node = qr_next(&arena->chunks_cache, cc_link); 1903 node != &arena->chunks_cache; node = qr_next(node, cc_link)) { 1904 qr_new(&node->rd, rd_link); 1905 qr_meld(&arena->runs_dirty, &node->rd, rd_link); 1906 } 1907 1908 /* Arena chunks. */ 1909 for (node = ql_last(&arena->achunks, ql_link); node != NULL; node = 1910 ql_last(&arena->achunks, ql_link)) { 1911 ql_remove(&arena->achunks, node, ql_link); 1912 arena_chunk_discard(tsd_tsdn(tsd), arena, 1913 extent_node_addr_get(node)); 1914 } 1915 1916 /* Spare. */ 1917 if (arena->spare != NULL) { 1918 arena_chunk_discard(tsd_tsdn(tsd), arena, arena->spare); 1919 arena->spare = NULL; 1920 } 1921 1922 assert(!arena->purging); 1923 arena->nactive = 0; 1924 1925 for (i = 0; i < sizeof(arena->runs_avail) / sizeof(arena_run_heap_t); 1926 i++) 1927 arena_run_heap_new(&arena->runs_avail[i]); 1928 1929 malloc_mutex_unlock(tsd_tsdn(tsd), &arena->lock); 1930} 1931 1932static void 1933arena_run_coalesce(arena_t *arena, arena_chunk_t *chunk, size_t *p_size, 1934 size_t *p_run_ind, size_t *p_run_pages, size_t flag_dirty, 1935 size_t flag_decommitted) 1936{ 1937 size_t size = *p_size; 1938 size_t run_ind = *p_run_ind; 1939 size_t run_pages = *p_run_pages; 1940 1941 /* Try to coalesce forward. */ 1942 if (run_ind + run_pages < chunk_npages && 1943 arena_mapbits_allocated_get(chunk, run_ind+run_pages) == 0 && 1944 arena_mapbits_dirty_get(chunk, run_ind+run_pages) == flag_dirty && 1945 arena_mapbits_decommitted_get(chunk, run_ind+run_pages) == 1946 flag_decommitted) { 1947 size_t nrun_size = arena_mapbits_unallocated_size_get(chunk, 1948 run_ind+run_pages); 1949 size_t nrun_pages = nrun_size >> LG_PAGE; 1950 1951 /* 1952 * Remove successor from runs_avail; the coalesced run is 1953 * inserted later. 1954 */ 1955 assert(arena_mapbits_unallocated_size_get(chunk, 1956 run_ind+run_pages+nrun_pages-1) == nrun_size); 1957 assert(arena_mapbits_dirty_get(chunk, 1958 run_ind+run_pages+nrun_pages-1) == flag_dirty); 1959 assert(arena_mapbits_decommitted_get(chunk, 1960 run_ind+run_pages+nrun_pages-1) == flag_decommitted); 1961 arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages); 1962 1963 /* 1964 * If the successor is dirty, remove it from the set of dirty 1965 * pages. 1966 */ 1967 if (flag_dirty != 0) { 1968 arena_run_dirty_remove(arena, chunk, run_ind+run_pages, 1969 nrun_pages); 1970 } 1971 1972 size += nrun_size; 1973 run_pages += nrun_pages; 1974 1975 arena_mapbits_unallocated_size_set(chunk, run_ind, size); 1976 arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, 1977 size); 1978 } 1979 1980 /* Try to coalesce backward. */ 1981 if (run_ind > map_bias && arena_mapbits_allocated_get(chunk, 1982 run_ind-1) == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) == 1983 flag_dirty && arena_mapbits_decommitted_get(chunk, run_ind-1) == 1984 flag_decommitted) { 1985 size_t prun_size = arena_mapbits_unallocated_size_get(chunk, 1986 run_ind-1); 1987 size_t prun_pages = prun_size >> LG_PAGE; 1988 1989 run_ind -= prun_pages; 1990 1991 /* 1992 * Remove predecessor from runs_avail; the coalesced run is 1993 * inserted later. 1994 */ 1995 assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == 1996 prun_size); 1997 assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty); 1998 assert(arena_mapbits_decommitted_get(chunk, run_ind) == 1999 flag_decommitted); 2000 arena_avail_remove(arena, chunk, run_ind, prun_pages); 2001 2002 /* 2003 * If the predecessor is dirty, remove it from the set of dirty 2004 * pages. 2005 */ 2006 if (flag_dirty != 0) { 2007 arena_run_dirty_remove(arena, chunk, run_ind, 2008 prun_pages); 2009 } 2010 2011 size += prun_size; 2012 run_pages += prun_pages; 2013 2014 arena_mapbits_unallocated_size_set(chunk, run_ind, size); 2015 arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, 2016 size); 2017 } 2018 2019 *p_size = size; 2020 *p_run_ind = run_ind; 2021 *p_run_pages = run_pages; 2022} 2023 2024static size_t 2025arena_run_size_get(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 2026 size_t run_ind) 2027{ 2028 size_t size; 2029 2030 assert(run_ind >= map_bias); 2031 assert(run_ind < chunk_npages); 2032 2033 if (arena_mapbits_large_get(chunk, run_ind) != 0) { 2034 size = arena_mapbits_large_size_get(chunk, run_ind); 2035 assert(size == PAGE || arena_mapbits_large_size_get(chunk, 2036 run_ind+(size>>LG_PAGE)-1) == 0); 2037 } else { 2038 arena_bin_info_t *bin_info = &arena_bin_info[run->binind]; 2039 size = bin_info->run_size; 2040 } 2041 2042 return (size); 2043} 2044 2045static void 2046arena_run_dalloc(tsdn_t *tsdn, arena_t *arena, arena_run_t *run, bool dirty, 2047 bool cleaned, bool decommitted) 2048{ 2049 arena_chunk_t *chunk; 2050 arena_chunk_map_misc_t *miscelm; 2051 size_t size, run_ind, run_pages, flag_dirty, flag_decommitted; 2052 2053 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 2054 miscelm = arena_run_to_miscelm(run); 2055 run_ind = arena_miscelm_to_pageind(miscelm); 2056 assert(run_ind >= map_bias); 2057 assert(run_ind < chunk_npages); 2058 size = arena_run_size_get(arena, chunk, run, run_ind); 2059 run_pages = (size >> LG_PAGE); 2060 arena_nactive_sub(arena, run_pages); 2061 2062 /* 2063 * The run is dirty if the caller claims to have dirtied it, as well as 2064 * if it was already dirty before being allocated and the caller 2065 * doesn't claim to have cleaned it. 2066 */ 2067 assert(arena_mapbits_dirty_get(chunk, run_ind) == 2068 arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); 2069 if (!cleaned && !decommitted && arena_mapbits_dirty_get(chunk, run_ind) 2070 != 0) 2071 dirty = true; 2072 flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; 2073 flag_decommitted = decommitted ? CHUNK_MAP_DECOMMITTED : 0; 2074 2075 /* Mark pages as unallocated in the chunk map. */ 2076 if (dirty || decommitted) { 2077 size_t flags = flag_dirty | flag_decommitted; 2078 arena_mapbits_unallocated_set(chunk, run_ind, size, flags); 2079 arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, 2080 flags); 2081 } else { 2082 arena_mapbits_unallocated_set(chunk, run_ind, size, 2083 arena_mapbits_unzeroed_get(chunk, run_ind)); 2084 arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, 2085 arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1)); 2086 } 2087 2088 arena_run_coalesce(arena, chunk, &size, &run_ind, &run_pages, 2089 flag_dirty, flag_decommitted); 2090 2091 /* Insert into runs_avail, now that coalescing is complete. */ 2092 assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == 2093 arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1)); 2094 assert(arena_mapbits_dirty_get(chunk, run_ind) == 2095 arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); 2096 assert(arena_mapbits_decommitted_get(chunk, run_ind) == 2097 arena_mapbits_decommitted_get(chunk, run_ind+run_pages-1)); 2098 arena_avail_insert(arena, chunk, run_ind, run_pages); 2099 2100 if (dirty) 2101 arena_run_dirty_insert(arena, chunk, run_ind, run_pages); 2102 2103 /* Deallocate chunk if it is now completely unused. */ 2104 if (size == arena_maxrun) { 2105 assert(run_ind == map_bias); 2106 assert(run_pages == (arena_maxrun >> LG_PAGE)); 2107 arena_chunk_dalloc(tsdn, arena, chunk); 2108 } 2109 2110 /* 2111 * It is okay to do dirty page processing here even if the chunk was 2112 * deallocated above, since in that case it is the spare. Waiting 2113 * until after possible chunk deallocation to do dirty processing 2114 * allows for an old spare to be fully deallocated, thus decreasing the 2115 * chances of spuriously crossing the dirty page purging threshold. 2116 */ 2117 if (dirty) 2118 arena_maybe_purge(tsdn, arena); 2119} 2120 2121static void 2122arena_run_trim_head(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, 2123 arena_run_t *run, size_t oldsize, size_t newsize) 2124{ 2125 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); 2126 size_t pageind = arena_miscelm_to_pageind(miscelm); 2127 size_t head_npages = (oldsize - newsize) >> LG_PAGE; 2128 size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); 2129 size_t flag_decommitted = arena_mapbits_decommitted_get(chunk, pageind); 2130 size_t flag_unzeroed_mask = (flag_dirty | flag_decommitted) == 0 ? 2131 CHUNK_MAP_UNZEROED : 0; 2132 2133 assert(oldsize > newsize); 2134 2135 /* 2136 * Update the chunk map so that arena_run_dalloc() can treat the 2137 * leading run as separately allocated. Set the last element of each 2138 * run first, in case of single-page runs. 2139 */ 2140 assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); 2141 arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty | 2142 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, 2143 pageind+head_npages-1))); 2144 arena_mapbits_large_set(chunk, pageind, oldsize-newsize, flag_dirty | 2145 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, pageind))); 2146 2147 if (config_debug) { 2148 UNUSED size_t tail_npages = newsize >> LG_PAGE; 2149 assert(arena_mapbits_large_size_get(chunk, 2150 pageind+head_npages+tail_npages-1) == 0); 2151 assert(arena_mapbits_dirty_get(chunk, 2152 pageind+head_npages+tail_npages-1) == flag_dirty); 2153 } 2154 arena_mapbits_large_set(chunk, pageind+head_npages, newsize, 2155 flag_dirty | (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, 2156 pageind+head_npages))); 2157 2158 arena_run_dalloc(tsdn, arena, run, false, false, (flag_decommitted != 2159 0)); 2160} 2161 2162static void 2163arena_run_trim_tail(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, 2164 arena_run_t *run, size_t oldsize, size_t newsize, bool dirty) 2165{ 2166 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); 2167 size_t pageind = arena_miscelm_to_pageind(miscelm); 2168 size_t head_npages = newsize >> LG_PAGE; 2169 size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); 2170 size_t flag_decommitted = arena_mapbits_decommitted_get(chunk, pageind); 2171 size_t flag_unzeroed_mask = (flag_dirty | flag_decommitted) == 0 ? 2172 CHUNK_MAP_UNZEROED : 0; 2173 arena_chunk_map_misc_t *tail_miscelm; 2174 arena_run_t *tail_run; 2175 2176 assert(oldsize > newsize); 2177 2178 /* 2179 * Update the chunk map so that arena_run_dalloc() can treat the 2180 * trailing run as separately allocated. Set the last element of each 2181 * run first, in case of single-page runs. 2182 */ 2183 assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); 2184 arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty | 2185 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, 2186 pageind+head_npages-1))); 2187 arena_mapbits_large_set(chunk, pageind, newsize, flag_dirty | 2188 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, pageind))); 2189 2190 if (config_debug) { 2191 UNUSED size_t tail_npages = (oldsize - newsize) >> LG_PAGE; 2192 assert(arena_mapbits_large_size_get(chunk, 2193 pageind+head_npages+tail_npages-1) == 0); 2194 assert(arena_mapbits_dirty_get(chunk, 2195 pageind+head_npages+tail_npages-1) == flag_dirty); 2196 } 2197 arena_mapbits_large_set(chunk, pageind+head_npages, oldsize-newsize, 2198 flag_dirty | (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, 2199 pageind+head_npages))); 2200 2201 tail_miscelm = arena_miscelm_get_mutable(chunk, pageind + head_npages); 2202 tail_run = &tail_miscelm->run; 2203 arena_run_dalloc(tsdn, arena, tail_run, dirty, false, (flag_decommitted 2204 != 0)); 2205} 2206 2207static void 2208arena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run) 2209{ 2210 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); 2211 2212 arena_run_heap_insert(&bin->runs, miscelm); 2213} 2214 2215static arena_run_t * 2216arena_bin_nonfull_run_tryget(arena_bin_t *bin) 2217{ 2218 arena_chunk_map_misc_t *miscelm; 2219 2220 miscelm = arena_run_heap_remove_first(&bin->runs); 2221 if (miscelm == NULL) 2222 return (NULL); 2223 if (config_stats) 2224 bin->stats.reruns++; 2225 2226 return (&miscelm->run); 2227} 2228 2229static arena_run_t * 2230arena_bin_nonfull_run_get(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin) 2231{ 2232 arena_run_t *run; 2233 szind_t binind; 2234 arena_bin_info_t *bin_info; 2235 2236 /* Look for a usable run. */ 2237 run = arena_bin_nonfull_run_tryget(bin); 2238 if (run != NULL) 2239 return (run); 2240 /* No existing runs have any space available. */ 2241 2242 binind = arena_bin_index(arena, bin); 2243 bin_info = &arena_bin_info[binind]; 2244 2245 /* Allocate a new run. */ 2246 malloc_mutex_unlock(tsdn, &bin->lock); 2247 /******************************/ 2248 malloc_mutex_lock(tsdn, &arena->lock); 2249 run = arena_run_alloc_small(tsdn, arena, bin_info->run_size, binind); 2250 if (run != NULL) { 2251 /* Initialize run internals. */ 2252 run->binind = binind; 2253 run->nfree = bin_info->nregs; 2254 bitmap_init(run->bitmap, &bin_info->bitmap_info); 2255 } 2256 malloc_mutex_unlock(tsdn, &arena->lock); 2257 /********************************/ 2258 malloc_mutex_lock(tsdn, &bin->lock); 2259 if (run != NULL) { 2260 if (config_stats) { 2261 bin->stats.nruns++; 2262 bin->stats.curruns++; 2263 } 2264 return (run); 2265 } 2266 2267 /* 2268 * arena_run_alloc_small() failed, but another thread may have made 2269 * sufficient memory available while this one dropped bin->lock above, 2270 * so search one more time. 2271 */ 2272 run = arena_bin_nonfull_run_tryget(bin); 2273 if (run != NULL) 2274 return (run); 2275 2276 return (NULL); 2277} 2278 2279/* Re-fill bin->runcur, then call arena_run_reg_alloc(). */ 2280static void * 2281arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin) 2282{ 2283 szind_t binind; 2284 arena_bin_info_t *bin_info; 2285 arena_run_t *run; 2286 2287 binind = arena_bin_index(arena, bin); 2288 bin_info = &arena_bin_info[binind]; 2289 bin->runcur = NULL; 2290 run = arena_bin_nonfull_run_get(tsdn, arena, bin); 2291 if (bin->runcur != NULL && bin->runcur->nfree > 0) { 2292 /* 2293 * Another thread updated runcur while this one ran without the 2294 * bin lock in arena_bin_nonfull_run_get(). 2295 */ 2296 void *ret; 2297 assert(bin->runcur->nfree > 0); 2298 ret = arena_run_reg_alloc(bin->runcur, bin_info); 2299 if (run != NULL) { 2300 arena_chunk_t *chunk; 2301 2302 /* 2303 * arena_run_alloc_small() may have allocated run, or 2304 * it may have pulled run from the bin's run tree. 2305 * Therefore it is unsafe to make any assumptions about 2306 * how run has previously been used, and 2307 * arena_bin_lower_run() must be called, as if a region 2308 * were just deallocated from the run. 2309 */ 2310 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 2311 if (run->nfree == bin_info->nregs) { 2312 arena_dalloc_bin_run(tsdn, arena, chunk, run, 2313 bin); 2314 } else 2315 arena_bin_lower_run(arena, chunk, run, bin); 2316 } 2317 return (ret); 2318 } 2319 2320 if (run == NULL) 2321 return (NULL); 2322 2323 bin->runcur = run; 2324 2325 assert(bin->runcur->nfree > 0); 2326 2327 return (arena_run_reg_alloc(bin->runcur, bin_info)); 2328} 2329 2330void 2331arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, tcache_bin_t *tbin, 2332 szind_t binind, uint64_t prof_accumbytes) 2333{ 2334 unsigned i, nfill; 2335 arena_bin_t *bin; 2336 2337 assert(tbin->ncached == 0); 2338 2339 if (config_prof && arena_prof_accum(tsdn, arena, prof_accumbytes)) 2340 prof_idump(tsdn); 2341 bin = &arena->bins[binind]; 2342 malloc_mutex_lock(tsdn, &bin->lock); 2343 for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> 2344 tbin->lg_fill_div); i < nfill; i++) { 2345 arena_run_t *run; 2346 void *ptr; 2347 if ((run = bin->runcur) != NULL && run->nfree > 0) 2348 ptr = arena_run_reg_alloc(run, &arena_bin_info[binind]); 2349 else 2350 ptr = arena_bin_malloc_hard(tsdn, arena, bin); 2351 if (ptr == NULL) { 2352 /* 2353 * OOM. tbin->avail isn't yet filled down to its first 2354 * element, so the successful allocations (if any) must 2355 * be moved just before tbin->avail before bailing out. 2356 */ 2357 if (i > 0) { 2358 memmove(tbin->avail - i, tbin->avail - nfill, 2359 i * sizeof(void *)); 2360 } 2361 break; 2362 } 2363 if (config_fill && unlikely(opt_junk_alloc)) { 2364 arena_alloc_junk_small(ptr, &arena_bin_info[binind], 2365 true); 2366 } 2367 /* Insert such that low regions get used first. */ 2368 *(tbin->avail - nfill + i) = ptr; 2369 } 2370 if (config_stats) { 2371 bin->stats.nmalloc += i; 2372 bin->stats.nrequests += tbin->tstats.nrequests; 2373 bin->stats.curregs += i; 2374 bin->stats.nfills++; 2375 tbin->tstats.nrequests = 0; 2376 } 2377 malloc_mutex_unlock(tsdn, &bin->lock); 2378 tbin->ncached = i; 2379 arena_decay_tick(tsdn, arena); 2380} 2381 2382void 2383arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, bool zero) 2384{ 2385 2386 size_t redzone_size = bin_info->redzone_size; 2387 2388 if (zero) { 2389 memset((void *)((uintptr_t)ptr - redzone_size), 2390 JEMALLOC_ALLOC_JUNK, redzone_size); 2391 memset((void *)((uintptr_t)ptr + bin_info->reg_size), 2392 JEMALLOC_ALLOC_JUNK, redzone_size); 2393 } else { 2394 memset((void *)((uintptr_t)ptr - redzone_size), 2395 JEMALLOC_ALLOC_JUNK, bin_info->reg_interval); 2396 } 2397} 2398 2399#ifdef JEMALLOC_JET 2400#undef arena_redzone_corruption 2401#define arena_redzone_corruption JEMALLOC_N(n_arena_redzone_corruption) 2402#endif 2403static void 2404arena_redzone_corruption(void *ptr, size_t usize, bool after, 2405 size_t offset, uint8_t byte) 2406{ 2407 2408 malloc_printf("<jemalloc>: Corrupt redzone %zu byte%s %s %p " 2409 "(size %zu), byte=%#x\n", offset, (offset == 1) ? "" : "s", 2410 after ? "after" : "before", ptr, usize, byte); 2411} 2412#ifdef JEMALLOC_JET 2413#undef arena_redzone_corruption 2414#define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption) 2415arena_redzone_corruption_t *arena_redzone_corruption = 2416 JEMALLOC_N(n_arena_redzone_corruption); 2417#endif 2418 2419static void 2420arena_redzones_validate(void *ptr, arena_bin_info_t *bin_info, bool reset) 2421{ 2422 bool error = false; 2423 2424 if (opt_junk_alloc) { 2425 size_t size = bin_info->reg_size; 2426 size_t redzone_size = bin_info->redzone_size; 2427 size_t i; 2428 2429 for (i = 1; i <= redzone_size; i++) { 2430 uint8_t *byte = (uint8_t *)((uintptr_t)ptr - i); 2431 if (*byte != JEMALLOC_ALLOC_JUNK) { 2432 error = true; 2433 arena_redzone_corruption(ptr, size, false, i, 2434 *byte); 2435 if (reset) 2436 *byte = JEMALLOC_ALLOC_JUNK; 2437 } 2438 } 2439 for (i = 0; i < redzone_size; i++) { 2440 uint8_t *byte = (uint8_t *)((uintptr_t)ptr + size + i); 2441 if (*byte != JEMALLOC_ALLOC_JUNK) { 2442 error = true; 2443 arena_redzone_corruption(ptr, size, true, i, 2444 *byte); 2445 if (reset) 2446 *byte = JEMALLOC_ALLOC_JUNK; 2447 } 2448 } 2449 } 2450 2451 if (opt_abort && error) 2452 abort(); 2453} 2454 2455#ifdef JEMALLOC_JET 2456#undef arena_dalloc_junk_small 2457#define arena_dalloc_junk_small JEMALLOC_N(n_arena_dalloc_junk_small) 2458#endif 2459void 2460arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info) 2461{ 2462 size_t redzone_size = bin_info->redzone_size; 2463 2464 arena_redzones_validate(ptr, bin_info, false); 2465 memset((void *)((uintptr_t)ptr - redzone_size), JEMALLOC_FREE_JUNK, 2466 bin_info->reg_interval); 2467} 2468#ifdef JEMALLOC_JET 2469#undef arena_dalloc_junk_small 2470#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small) 2471arena_dalloc_junk_small_t *arena_dalloc_junk_small = 2472 JEMALLOC_N(n_arena_dalloc_junk_small); 2473#endif 2474 2475void 2476arena_quarantine_junk_small(void *ptr, size_t usize) 2477{ 2478 szind_t binind; 2479 arena_bin_info_t *bin_info; 2480 cassert(config_fill); 2481 assert(opt_junk_free); 2482 assert(opt_quarantine); 2483 assert(usize <= SMALL_MAXCLASS); 2484 2485 binind = size2index(usize); 2486 bin_info = &arena_bin_info[binind]; 2487 arena_redzones_validate(ptr, bin_info, true); 2488} 2489 2490static void * 2491arena_malloc_small(tsdn_t *tsdn, arena_t *arena, szind_t binind, bool zero) 2492{ 2493 void *ret; 2494 arena_bin_t *bin; 2495 size_t usize; 2496 arena_run_t *run; 2497 2498 assert(binind < NBINS); 2499 bin = &arena->bins[binind]; 2500 usize = index2size(binind); 2501 2502 malloc_mutex_lock(tsdn, &bin->lock); 2503 if ((run = bin->runcur) != NULL && run->nfree > 0) 2504 ret = arena_run_reg_alloc(run, &arena_bin_info[binind]); 2505 else 2506 ret = arena_bin_malloc_hard(tsdn, arena, bin); 2507 2508 if (ret == NULL) { 2509 malloc_mutex_unlock(tsdn, &bin->lock); 2510 return (NULL); 2511 } 2512 2513 if (config_stats) { 2514 bin->stats.nmalloc++; 2515 bin->stats.nrequests++; 2516 bin->stats.curregs++; 2517 } 2518 malloc_mutex_unlock(tsdn, &bin->lock); 2519 if (config_prof && !isthreaded && arena_prof_accum(tsdn, arena, usize)) 2520 prof_idump(tsdn); 2521 2522 if (!zero) { 2523 if (config_fill) { 2524 if (unlikely(opt_junk_alloc)) { 2525 arena_alloc_junk_small(ret, 2526 &arena_bin_info[binind], false); 2527 } else if (unlikely(opt_zero)) 2528 memset(ret, 0, usize); 2529 } 2530 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, usize); 2531 } else { 2532 if (config_fill && unlikely(opt_junk_alloc)) { 2533 arena_alloc_junk_small(ret, &arena_bin_info[binind], 2534 true); 2535 } 2536 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, usize); 2537 memset(ret, 0, usize); 2538 } 2539 2540 arena_decay_tick(tsdn, arena); 2541 return (ret); 2542} 2543 2544void * 2545arena_malloc_large(tsdn_t *tsdn, arena_t *arena, szind_t binind, bool zero) 2546{ 2547 void *ret; 2548 size_t usize; 2549 uintptr_t random_offset; 2550 arena_run_t *run; 2551 arena_chunk_map_misc_t *miscelm; 2552 UNUSED bool idump JEMALLOC_CC_SILENCE_INIT(false); 2553 2554 /* Large allocation. */ 2555 usize = index2size(binind); 2556 malloc_mutex_lock(tsdn, &arena->lock); 2557 if (config_cache_oblivious) { 2558 uint64_t r; 2559 2560 /* 2561 * Compute a uniformly distributed offset within the first page 2562 * that is a multiple of the cacheline size, e.g. [0 .. 63) * 64 2563 * for 4 KiB pages and 64-byte cachelines. 2564 */ 2565 r = prng_lg_range(&arena->offset_state, LG_PAGE - LG_CACHELINE); 2566 random_offset = ((uintptr_t)r) << LG_CACHELINE; 2567 } else 2568 random_offset = 0; 2569 run = arena_run_alloc_large(tsdn, arena, usize + large_pad, zero); 2570 if (run == NULL) { 2571 malloc_mutex_unlock(tsdn, &arena->lock); 2572 return (NULL); 2573 } 2574 miscelm = arena_run_to_miscelm(run); 2575 ret = (void *)((uintptr_t)arena_miscelm_to_rpages(miscelm) + 2576 random_offset); 2577 if (config_stats) { 2578 szind_t index = binind - NBINS; 2579 2580 arena->stats.nmalloc_large++; 2581 arena->stats.nrequests_large++; 2582 arena->stats.allocated_large += usize; 2583 arena->stats.lstats[index].nmalloc++; 2584 arena->stats.lstats[index].nrequests++; 2585 arena->stats.lstats[index].curruns++; 2586 } 2587 if (config_prof) 2588 idump = arena_prof_accum_locked(arena, usize); 2589 malloc_mutex_unlock(tsdn, &arena->lock); 2590 if (config_prof && idump) 2591 prof_idump(tsdn); 2592 2593 if (!zero) { 2594 if (config_fill) { 2595 if (unlikely(opt_junk_alloc)) 2596 memset(ret, JEMALLOC_ALLOC_JUNK, usize); 2597 else if (unlikely(opt_zero)) 2598 memset(ret, 0, usize); 2599 } 2600 } 2601 2602 arena_decay_tick(tsdn, arena); 2603 return (ret); 2604} 2605 2606void * 2607arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, 2608 bool zero) 2609{ 2610 2611 assert(!tsdn_null(tsdn) || arena != NULL); 2612 2613 if (likely(!tsdn_null(tsdn))) 2614 arena = arena_choose(tsdn_tsd(tsdn), arena); 2615 if (unlikely(arena == NULL)) 2616 return (NULL); 2617 2618 if (likely(size <= SMALL_MAXCLASS)) 2619 return (arena_malloc_small(tsdn, arena, ind, zero)); 2620 if (likely(size <= large_maxclass)) 2621 return (arena_malloc_large(tsdn, arena, ind, zero)); 2622 return (huge_malloc(tsdn, arena, index2size(ind), zero)); 2623} 2624 2625/* Only handles large allocations that require more than page alignment. */ 2626static void * 2627arena_palloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, 2628 bool zero) 2629{ 2630 void *ret; 2631 size_t alloc_size, leadsize, trailsize; 2632 arena_run_t *run; 2633 arena_chunk_t *chunk; 2634 arena_chunk_map_misc_t *miscelm; 2635 void *rpages; 2636 2637 assert(!tsdn_null(tsdn) || arena != NULL); 2638 assert(usize == PAGE_CEILING(usize)); 2639 2640 if (likely(!tsdn_null(tsdn))) 2641 arena = arena_choose(tsdn_tsd(tsdn), arena); 2642 if (unlikely(arena == NULL)) 2643 return (NULL); 2644 2645 alignment = PAGE_CEILING(alignment); 2646 alloc_size = usize + large_pad + alignment - PAGE; 2647 2648 malloc_mutex_lock(tsdn, &arena->lock); 2649 run = arena_run_alloc_large(tsdn, arena, alloc_size, false); 2650 if (run == NULL) { 2651 malloc_mutex_unlock(tsdn, &arena->lock); 2652 return (NULL); 2653 } 2654 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 2655 miscelm = arena_run_to_miscelm(run); 2656 rpages = arena_miscelm_to_rpages(miscelm); 2657 2658 leadsize = ALIGNMENT_CEILING((uintptr_t)rpages, alignment) - 2659 (uintptr_t)rpages; 2660 assert(alloc_size >= leadsize + usize); 2661 trailsize = alloc_size - leadsize - usize - large_pad; 2662 if (leadsize != 0) { 2663 arena_chunk_map_misc_t *head_miscelm = miscelm; 2664 arena_run_t *head_run = run; 2665 2666 miscelm = arena_miscelm_get_mutable(chunk, 2667 arena_miscelm_to_pageind(head_miscelm) + (leadsize >> 2668 LG_PAGE)); 2669 run = &miscelm->run; 2670 2671 arena_run_trim_head(tsdn, arena, chunk, head_run, alloc_size, 2672 alloc_size - leadsize); 2673 } 2674 if (trailsize != 0) { 2675 arena_run_trim_tail(tsdn, arena, chunk, run, usize + large_pad + 2676 trailsize, usize + large_pad, false); 2677 } 2678 if (arena_run_init_large(arena, run, usize + large_pad, zero)) { 2679 size_t run_ind = 2680 arena_miscelm_to_pageind(arena_run_to_miscelm(run)); 2681 bool dirty = (arena_mapbits_dirty_get(chunk, run_ind) != 0); 2682 bool decommitted = (arena_mapbits_decommitted_get(chunk, 2683 run_ind) != 0); 2684 2685 assert(decommitted); /* Cause of OOM. */ 2686 arena_run_dalloc(tsdn, arena, run, dirty, false, decommitted); 2687 malloc_mutex_unlock(tsdn, &arena->lock); 2688 return (NULL); 2689 } 2690 ret = arena_miscelm_to_rpages(miscelm); 2691 2692 if (config_stats) { 2693 szind_t index = size2index(usize) - NBINS; 2694 2695 arena->stats.nmalloc_large++; 2696 arena->stats.nrequests_large++; 2697 arena->stats.allocated_large += usize; 2698 arena->stats.lstats[index].nmalloc++; 2699 arena->stats.lstats[index].nrequests++; 2700 arena->stats.lstats[index].curruns++; 2701 } 2702 malloc_mutex_unlock(tsdn, &arena->lock); 2703 2704 if (config_fill && !zero) { 2705 if (unlikely(opt_junk_alloc)) 2706 memset(ret, JEMALLOC_ALLOC_JUNK, usize); 2707 else if (unlikely(opt_zero)) 2708 memset(ret, 0, usize); 2709 } 2710 arena_decay_tick(tsdn, arena); 2711 return (ret); 2712} 2713 2714void * 2715arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, 2716 bool zero, tcache_t *tcache) 2717{ 2718 void *ret; 2719 2720 if (usize <= SMALL_MAXCLASS && (alignment < PAGE || (alignment == PAGE 2721 && (usize & PAGE_MASK) == 0))) { 2722 /* Small; alignment doesn't require special run placement. */ 2723 ret = arena_malloc(tsdn, arena, usize, size2index(usize), zero, 2724 tcache, true); 2725 } else if (usize <= large_maxclass && alignment <= PAGE) { 2726 /* 2727 * Large; alignment doesn't require special run placement. 2728 * However, the cached pointer may be at a random offset from 2729 * the base of the run, so do some bit manipulation to retrieve 2730 * the base. 2731 */ 2732 ret = arena_malloc(tsdn, arena, usize, size2index(usize), zero, 2733 tcache, true); 2734 if (config_cache_oblivious) 2735 ret = (void *)((uintptr_t)ret & ~PAGE_MASK); 2736 } else { 2737 if (likely(usize <= large_maxclass)) { 2738 ret = arena_palloc_large(tsdn, arena, usize, alignment, 2739 zero); 2740 } else if (likely(alignment <= chunksize)) 2741 ret = huge_malloc(tsdn, arena, usize, zero); 2742 else { 2743 ret = huge_palloc(tsdn, arena, usize, alignment, zero); 2744 } 2745 } 2746 return (ret); 2747} 2748 2749void 2750arena_prof_promoted(tsdn_t *tsdn, const void *ptr, size_t size) 2751{ 2752 arena_chunk_t *chunk; 2753 size_t pageind; 2754 szind_t binind; 2755 2756 cassert(config_prof); 2757 assert(ptr != NULL); 2758 assert(CHUNK_ADDR2BASE(ptr) != ptr); 2759 assert(isalloc(tsdn, ptr, false) == LARGE_MINCLASS); 2760 assert(isalloc(tsdn, ptr, true) == LARGE_MINCLASS); 2761 assert(size <= SMALL_MAXCLASS); 2762 2763 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 2764 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 2765 binind = size2index(size); 2766 assert(binind < NBINS); 2767 arena_mapbits_large_binind_set(chunk, pageind, binind); 2768 2769 assert(isalloc(tsdn, ptr, false) == LARGE_MINCLASS); 2770 assert(isalloc(tsdn, ptr, true) == size); 2771} 2772 2773static void 2774arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, 2775 arena_bin_t *bin) 2776{ 2777 2778 /* Dissociate run from bin. */ 2779 if (run == bin->runcur) 2780 bin->runcur = NULL; 2781 else { 2782 szind_t binind = arena_bin_index(extent_node_arena_get( 2783 &chunk->node), bin); 2784 arena_bin_info_t *bin_info = &arena_bin_info[binind]; 2785 2786 /* 2787 * The following block's conditional is necessary because if the 2788 * run only contains one region, then it never gets inserted 2789 * into the non-full runs tree. 2790 */ 2791 if (bin_info->nregs != 1) { 2792 arena_chunk_map_misc_t *miscelm = 2793 arena_run_to_miscelm(run); 2794 2795 arena_run_heap_remove(&bin->runs, miscelm); 2796 } 2797 } 2798} 2799 2800static void 2801arena_dalloc_bin_run(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, 2802 arena_run_t *run, arena_bin_t *bin) 2803{ 2804 2805 assert(run != bin->runcur); 2806 2807 malloc_mutex_unlock(tsdn, &bin->lock); 2808 /******************************/ 2809 malloc_mutex_lock(tsdn, &arena->lock); 2810 arena_run_dalloc(tsdn, arena, run, true, false, false); 2811 malloc_mutex_unlock(tsdn, &arena->lock); 2812 /****************************/ 2813 malloc_mutex_lock(tsdn, &bin->lock); 2814 if (config_stats) 2815 bin->stats.curruns--; 2816} 2817 2818static void 2819arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 2820 arena_bin_t *bin) 2821{ 2822 2823 /* 2824 * Make sure that if bin->runcur is non-NULL, it refers to the lowest 2825 * non-full run. It is okay to NULL runcur out rather than proactively 2826 * keeping it pointing at the lowest non-full run. 2827 */ 2828 if ((uintptr_t)run < (uintptr_t)bin->runcur) { 2829 /* Switch runcur. */ 2830 if (bin->runcur->nfree > 0) 2831 arena_bin_runs_insert(bin, bin->runcur); 2832 bin->runcur = run; 2833 if (config_stats) 2834 bin->stats.reruns++; 2835 } else 2836 arena_bin_runs_insert(bin, run); 2837} 2838 2839static void 2840arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, 2841 void *ptr, arena_chunk_map_bits_t *bitselm, bool junked) 2842{ 2843 size_t pageind, rpages_ind; 2844 arena_run_t *run; 2845 arena_bin_t *bin; 2846 arena_bin_info_t *bin_info; 2847 szind_t binind; 2848 2849 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 2850 rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind); 2851 run = &arena_miscelm_get_mutable(chunk, rpages_ind)->run; 2852 binind = run->binind; 2853 bin = &arena->bins[binind]; 2854 bin_info = &arena_bin_info[binind]; 2855 2856 if (!junked && config_fill && unlikely(opt_junk_free)) 2857 arena_dalloc_junk_small(ptr, bin_info); 2858 2859 arena_run_reg_dalloc(run, ptr); 2860 if (run->nfree == bin_info->nregs) { 2861 arena_dissociate_bin_run(chunk, run, bin); 2862 arena_dalloc_bin_run(tsdn, arena, chunk, run, bin); 2863 } else if (run->nfree == 1 && run != bin->runcur) 2864 arena_bin_lower_run(arena, chunk, run, bin); 2865 2866 if (config_stats) { 2867 bin->stats.ndalloc++; 2868 bin->stats.curregs--; 2869 } 2870} 2871 2872void 2873arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, 2874 arena_chunk_t *chunk, void *ptr, arena_chunk_map_bits_t *bitselm) 2875{ 2876 2877 arena_dalloc_bin_locked_impl(tsdn, arena, chunk, ptr, bitselm, true); 2878} 2879 2880void 2881arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, void *ptr, 2882 size_t pageind, arena_chunk_map_bits_t *bitselm) 2883{ 2884 arena_run_t *run; 2885 arena_bin_t *bin; 2886 size_t rpages_ind; 2887 2888 rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind); 2889 run = &arena_miscelm_get_mutable(chunk, rpages_ind)->run; 2890 bin = &arena->bins[run->binind]; 2891 malloc_mutex_lock(tsdn, &bin->lock); 2892 arena_dalloc_bin_locked_impl(tsdn, arena, chunk, ptr, bitselm, false); 2893 malloc_mutex_unlock(tsdn, &bin->lock); 2894} 2895 2896void 2897arena_dalloc_small(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, 2898 void *ptr, size_t pageind) 2899{ 2900 arena_chunk_map_bits_t *bitselm; 2901 2902 if (config_debug) { 2903 /* arena_ptr_small_binind_get() does extra sanity checking. */ 2904 assert(arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, 2905 pageind)) != BININD_INVALID); 2906 } 2907 bitselm = arena_bitselm_get_mutable(chunk, pageind); 2908 arena_dalloc_bin(tsdn, arena, chunk, ptr, pageind, bitselm); 2909 arena_decay_tick(tsdn, arena); 2910} 2911 2912#ifdef JEMALLOC_JET 2913#undef arena_dalloc_junk_large 2914#define arena_dalloc_junk_large JEMALLOC_N(n_arena_dalloc_junk_large) 2915#endif 2916void 2917arena_dalloc_junk_large(void *ptr, size_t usize) 2918{ 2919 2920 if (config_fill && unlikely(opt_junk_free)) 2921 memset(ptr, JEMALLOC_FREE_JUNK, usize); 2922} 2923#ifdef JEMALLOC_JET 2924#undef arena_dalloc_junk_large 2925#define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large) 2926arena_dalloc_junk_large_t *arena_dalloc_junk_large = 2927 JEMALLOC_N(n_arena_dalloc_junk_large); 2928#endif 2929 2930static void 2931arena_dalloc_large_locked_impl(tsdn_t *tsdn, arena_t *arena, 2932 arena_chunk_t *chunk, void *ptr, bool junked) 2933{ 2934 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 2935 arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk, 2936 pageind); 2937 arena_run_t *run = &miscelm->run; 2938 2939 if (config_fill || config_stats) { 2940 size_t usize = arena_mapbits_large_size_get(chunk, pageind) - 2941 large_pad; 2942 2943 if (!junked) 2944 arena_dalloc_junk_large(ptr, usize); 2945 if (config_stats) { 2946 szind_t index = size2index(usize) - NBINS; 2947 2948 arena->stats.ndalloc_large++; 2949 arena->stats.allocated_large -= usize; 2950 arena->stats.lstats[index].ndalloc++; 2951 arena->stats.lstats[index].curruns--; 2952 } 2953 } 2954 2955 arena_run_dalloc(tsdn, arena, run, true, false, false); 2956} 2957 2958void 2959arena_dalloc_large_junked_locked(tsdn_t *tsdn, arena_t *arena, 2960 arena_chunk_t *chunk, void *ptr) 2961{ 2962 2963 arena_dalloc_large_locked_impl(tsdn, arena, chunk, ptr, true); 2964} 2965 2966void 2967arena_dalloc_large(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, 2968 void *ptr) 2969{ 2970 2971 malloc_mutex_lock(tsdn, &arena->lock); 2972 arena_dalloc_large_locked_impl(tsdn, arena, chunk, ptr, false); 2973 malloc_mutex_unlock(tsdn, &arena->lock); 2974 arena_decay_tick(tsdn, arena); 2975} 2976 2977static void 2978arena_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, 2979 void *ptr, size_t oldsize, size_t size) 2980{ 2981 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 2982 arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk, 2983 pageind); 2984 arena_run_t *run = &miscelm->run; 2985 2986 assert(size < oldsize); 2987 2988 /* 2989 * Shrink the run, and make trailing pages available for other 2990 * allocations. 2991 */ 2992 malloc_mutex_lock(tsdn, &arena->lock); 2993 arena_run_trim_tail(tsdn, arena, chunk, run, oldsize + large_pad, size + 2994 large_pad, true); 2995 if (config_stats) { 2996 szind_t oldindex = size2index(oldsize) - NBINS; 2997 szind_t index = size2index(size) - NBINS; 2998 2999 arena->stats.ndalloc_large++; 3000 arena->stats.allocated_large -= oldsize; 3001 arena->stats.lstats[oldindex].ndalloc++; 3002 arena->stats.lstats[oldindex].curruns--; 3003 3004 arena->stats.nmalloc_large++; 3005 arena->stats.nrequests_large++; 3006 arena->stats.allocated_large += size; 3007 arena->stats.lstats[index].nmalloc++; 3008 arena->stats.lstats[index].nrequests++; 3009 arena->stats.lstats[index].curruns++; 3010 } 3011 malloc_mutex_unlock(tsdn, &arena->lock); 3012} 3013 3014static bool 3015arena_ralloc_large_grow(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, 3016 void *ptr, size_t oldsize, size_t usize_min, size_t usize_max, bool zero) 3017{ 3018 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 3019 size_t npages = (oldsize + large_pad) >> LG_PAGE; 3020 size_t followsize; 3021 3022 assert(oldsize == arena_mapbits_large_size_get(chunk, pageind) - 3023 large_pad); 3024 3025 /* Try to extend the run. */ 3026 malloc_mutex_lock(tsdn, &arena->lock); 3027 if (pageind+npages >= chunk_npages || arena_mapbits_allocated_get(chunk, 3028 pageind+npages) != 0) 3029 goto label_fail; 3030 followsize = arena_mapbits_unallocated_size_get(chunk, pageind+npages); 3031 if (oldsize + followsize >= usize_min) { 3032 /* 3033 * The next run is available and sufficiently large. Split the 3034 * following run, then merge the first part with the existing 3035 * allocation. 3036 */ 3037 arena_run_t *run; 3038 size_t usize, splitsize, size, flag_dirty, flag_unzeroed_mask; 3039 3040 usize = usize_max; 3041 while (oldsize + followsize < usize) 3042 usize = index2size(size2index(usize)-1); 3043 assert(usize >= usize_min); 3044 assert(usize >= oldsize); 3045 splitsize = usize - oldsize; 3046 if (splitsize == 0) 3047 goto label_fail; 3048 3049 run = &arena_miscelm_get_mutable(chunk, pageind+npages)->run; 3050 if (arena_run_split_large(arena, run, splitsize, zero)) 3051 goto label_fail; 3052 3053 if (config_cache_oblivious && zero) { 3054 /* 3055 * Zero the trailing bytes of the original allocation's 3056 * last page, since they are in an indeterminate state. 3057 * There will always be trailing bytes, because ptr's 3058 * offset from the beginning of the run is a multiple of 3059 * CACHELINE in [0 .. PAGE). 3060 */ 3061 void *zbase = (void *)((uintptr_t)ptr + oldsize); 3062 void *zpast = PAGE_ADDR2BASE((void *)((uintptr_t)zbase + 3063 PAGE)); 3064 size_t nzero = (uintptr_t)zpast - (uintptr_t)zbase; 3065 assert(nzero > 0); 3066 memset(zbase, 0, nzero); 3067 } 3068 3069 size = oldsize + splitsize; 3070 npages = (size + large_pad) >> LG_PAGE; 3071 3072 /* 3073 * Mark the extended run as dirty if either portion of the run 3074 * was dirty before allocation. This is rather pedantic, 3075 * because there's not actually any sequence of events that 3076 * could cause the resulting run to be passed to 3077 * arena_run_dalloc() with the dirty argument set to false 3078 * (which is when dirty flag consistency would really matter). 3079 */ 3080 flag_dirty = arena_mapbits_dirty_get(chunk, pageind) | 3081 arena_mapbits_dirty_get(chunk, pageind+npages-1); 3082 flag_unzeroed_mask = flag_dirty == 0 ? CHUNK_MAP_UNZEROED : 0; 3083 arena_mapbits_large_set(chunk, pageind, size + large_pad, 3084 flag_dirty | (flag_unzeroed_mask & 3085 arena_mapbits_unzeroed_get(chunk, pageind))); 3086 arena_mapbits_large_set(chunk, pageind+npages-1, 0, flag_dirty | 3087 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, 3088 pageind+npages-1))); 3089 3090 if (config_stats) { 3091 szind_t oldindex = size2index(oldsize) - NBINS; 3092 szind_t index = size2index(size) - NBINS; 3093 3094 arena->stats.ndalloc_large++; 3095 arena->stats.allocated_large -= oldsize; 3096 arena->stats.lstats[oldindex].ndalloc++; 3097 arena->stats.lstats[oldindex].curruns--; 3098 3099 arena->stats.nmalloc_large++; 3100 arena->stats.nrequests_large++; 3101 arena->stats.allocated_large += size; 3102 arena->stats.lstats[index].nmalloc++; 3103 arena->stats.lstats[index].nrequests++; 3104 arena->stats.lstats[index].curruns++; 3105 } 3106 malloc_mutex_unlock(tsdn, &arena->lock); 3107 return (false); 3108 } 3109label_fail: 3110 malloc_mutex_unlock(tsdn, &arena->lock); 3111 return (true); 3112} 3113 3114#ifdef JEMALLOC_JET 3115#undef arena_ralloc_junk_large 3116#define arena_ralloc_junk_large JEMALLOC_N(n_arena_ralloc_junk_large) 3117#endif 3118static void 3119arena_ralloc_junk_large(void *ptr, size_t old_usize, size_t usize) 3120{ 3121 3122 if (config_fill && unlikely(opt_junk_free)) { 3123 memset((void *)((uintptr_t)ptr + usize), JEMALLOC_FREE_JUNK, 3124 old_usize - usize); 3125 } 3126} 3127#ifdef JEMALLOC_JET 3128#undef arena_ralloc_junk_large 3129#define arena_ralloc_junk_large JEMALLOC_N(arena_ralloc_junk_large) 3130arena_ralloc_junk_large_t *arena_ralloc_junk_large = 3131 JEMALLOC_N(n_arena_ralloc_junk_large); 3132#endif 3133 3134/* 3135 * Try to resize a large allocation, in order to avoid copying. This will 3136 * always fail if growing an object, and the following run is already in use. 3137 */ 3138static bool 3139arena_ralloc_large(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min, 3140 size_t usize_max, bool zero) 3141{ 3142 arena_chunk_t *chunk; 3143 arena_t *arena; 3144 3145 if (oldsize == usize_max) { 3146 /* Current size class is compatible and maximal. */ 3147 return (false); 3148 } 3149 3150 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 3151 arena = extent_node_arena_get(&chunk->node); 3152 3153 if (oldsize < usize_max) { 3154 bool ret = arena_ralloc_large_grow(tsdn, arena, chunk, ptr, 3155 oldsize, usize_min, usize_max, zero); 3156 if (config_fill && !ret && !zero) { 3157 if (unlikely(opt_junk_alloc)) { 3158 memset((void *)((uintptr_t)ptr + oldsize), 3159 JEMALLOC_ALLOC_JUNK, 3160 isalloc(tsdn, ptr, config_prof) - oldsize); 3161 } else if (unlikely(opt_zero)) { 3162 memset((void *)((uintptr_t)ptr + oldsize), 0, 3163 isalloc(tsdn, ptr, config_prof) - oldsize); 3164 } 3165 } 3166 return (ret); 3167 } 3168 3169 assert(oldsize > usize_max); 3170 /* Fill before shrinking in order avoid a race. */ 3171 arena_ralloc_junk_large(ptr, oldsize, usize_max); 3172 arena_ralloc_large_shrink(tsdn, arena, chunk, ptr, oldsize, usize_max); 3173 return (false); 3174} 3175 3176bool 3177arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, 3178 size_t extra, bool zero) 3179{ 3180 size_t usize_min, usize_max; 3181 3182 /* Calls with non-zero extra had to clamp extra. */ 3183 assert(extra == 0 || size + extra <= HUGE_MAXCLASS); 3184 3185 if (unlikely(size > HUGE_MAXCLASS)) 3186 return (true); 3187 3188 usize_min = s2u(size); 3189 usize_max = s2u(size + extra); 3190 if (likely(oldsize <= large_maxclass && usize_min <= large_maxclass)) { 3191 arena_chunk_t *chunk; 3192 3193 /* 3194 * Avoid moving the allocation if the size class can be left the 3195 * same. 3196 */ 3197 if (oldsize <= SMALL_MAXCLASS) { 3198 assert(arena_bin_info[size2index(oldsize)].reg_size == 3199 oldsize); 3200 if ((usize_max > SMALL_MAXCLASS || 3201 size2index(usize_max) != size2index(oldsize)) && 3202 (size > oldsize || usize_max < oldsize)) 3203 return (true); 3204 } else { 3205 if (usize_max <= SMALL_MAXCLASS) 3206 return (true); 3207 if (arena_ralloc_large(tsdn, ptr, oldsize, usize_min, 3208 usize_max, zero)) 3209 return (true); 3210 } 3211 3212 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 3213 arena_decay_tick(tsdn, extent_node_arena_get(&chunk->node)); 3214 return (false); 3215 } else { 3216 return (huge_ralloc_no_move(tsdn, ptr, oldsize, usize_min, 3217 usize_max, zero)); 3218 } 3219} 3220 3221static void * 3222arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize, 3223 size_t alignment, bool zero, tcache_t *tcache) 3224{ 3225 3226 if (alignment == 0) 3227 return (arena_malloc(tsdn, arena, usize, size2index(usize), 3228 zero, tcache, true)); 3229 usize = sa2u(usize, alignment); 3230 if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) 3231 return (NULL); 3232 return (ipalloct(tsdn, usize, alignment, zero, tcache, arena)); 3233} 3234 3235void * 3236arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, 3237 size_t alignment, bool zero, tcache_t *tcache) 3238{ 3239 void *ret; 3240 size_t usize; 3241 3242 usize = s2u(size); 3243 if (unlikely(usize == 0 || size > HUGE_MAXCLASS)) 3244 return (NULL); 3245 3246 if (likely(usize <= large_maxclass)) { 3247 size_t copysize; 3248 3249 /* Try to avoid moving the allocation. */ 3250 if (!arena_ralloc_no_move(tsd_tsdn(tsd), ptr, oldsize, usize, 0, 3251 zero)) 3252 return (ptr); 3253 3254 /* 3255 * size and oldsize are different enough that we need to move 3256 * the object. In that case, fall back to allocating new space 3257 * and copying. 3258 */ 3259 ret = arena_ralloc_move_helper(tsd_tsdn(tsd), arena, usize, 3260 alignment, zero, tcache); 3261 if (ret == NULL) 3262 return (NULL); 3263 3264 /* 3265 * Junk/zero-filling were already done by 3266 * ipalloc()/arena_malloc(). 3267 */ 3268 3269 copysize = (usize < oldsize) ? usize : oldsize; 3270 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize); 3271 memcpy(ret, ptr, copysize); 3272 isqalloc(tsd, ptr, oldsize, tcache, true); 3273 } else { 3274 ret = huge_ralloc(tsd, arena, ptr, oldsize, usize, alignment, 3275 zero, tcache); 3276 } 3277 return (ret); 3278} 3279 3280dss_prec_t 3281arena_dss_prec_get(tsdn_t *tsdn, arena_t *arena) 3282{ 3283 dss_prec_t ret; 3284 3285 malloc_mutex_lock(tsdn, &arena->lock); 3286 ret = arena->dss_prec; 3287 malloc_mutex_unlock(tsdn, &arena->lock); 3288 return (ret); 3289} 3290 3291bool 3292arena_dss_prec_set(tsdn_t *tsdn, arena_t *arena, dss_prec_t dss_prec) 3293{ 3294 3295 if (!have_dss) 3296 return (dss_prec != dss_prec_disabled); 3297 malloc_mutex_lock(tsdn, &arena->lock); 3298 arena->dss_prec = dss_prec; 3299 malloc_mutex_unlock(tsdn, &arena->lock); 3300 return (false); 3301} 3302 3303ssize_t 3304arena_lg_dirty_mult_default_get(void) 3305{ 3306 3307 return ((ssize_t)atomic_read_z((size_t *)&lg_dirty_mult_default)); 3308} 3309 3310bool 3311arena_lg_dirty_mult_default_set(ssize_t lg_dirty_mult) 3312{ 3313 3314 if (opt_purge != purge_mode_ratio) 3315 return (true); 3316 if (!arena_lg_dirty_mult_valid(lg_dirty_mult)) 3317 return (true); 3318 atomic_write_z((size_t *)&lg_dirty_mult_default, (size_t)lg_dirty_mult); 3319 return (false); 3320} 3321 3322ssize_t 3323arena_decay_time_default_get(void) 3324{ 3325 3326 return ((ssize_t)atomic_read_z((size_t *)&decay_time_default)); 3327} 3328 3329bool 3330arena_decay_time_default_set(ssize_t decay_time) 3331{ 3332 3333 if (opt_purge != purge_mode_decay) 3334 return (true); 3335 if (!arena_decay_time_valid(decay_time)) 3336 return (true); 3337 atomic_write_z((size_t *)&decay_time_default, (size_t)decay_time); 3338 return (false); 3339} 3340 3341static void 3342arena_basic_stats_merge_locked(arena_t *arena, unsigned *nthreads, 3343 const char **dss, ssize_t *lg_dirty_mult, ssize_t *decay_time, 3344 size_t *nactive, size_t *ndirty) 3345{ 3346 3347 *nthreads += arena_nthreads_get(arena, false); 3348 *dss = dss_prec_names[arena->dss_prec]; 3349 *lg_dirty_mult = arena->lg_dirty_mult; 3350 *decay_time = arena->decay.time; 3351 *nactive += arena->nactive; 3352 *ndirty += arena->ndirty; 3353} 3354 3355void 3356arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, 3357 const char **dss, ssize_t *lg_dirty_mult, ssize_t *decay_time, 3358 size_t *nactive, size_t *ndirty) 3359{ 3360 3361 malloc_mutex_lock(tsdn, &arena->lock); 3362 arena_basic_stats_merge_locked(arena, nthreads, dss, lg_dirty_mult, 3363 decay_time, nactive, ndirty); 3364 malloc_mutex_unlock(tsdn, &arena->lock); 3365} 3366 3367void 3368arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, 3369 const char **dss, ssize_t *lg_dirty_mult, ssize_t *decay_time, 3370 size_t *nactive, size_t *ndirty, arena_stats_t *astats, 3371 malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats, 3372 malloc_huge_stats_t *hstats) 3373{ 3374 unsigned i; 3375 3376 cassert(config_stats); 3377 3378 malloc_mutex_lock(tsdn, &arena->lock); 3379 arena_basic_stats_merge_locked(arena, nthreads, dss, lg_dirty_mult, 3380 decay_time, nactive, ndirty); 3381 3382 astats->mapped += arena->stats.mapped; 3383 astats->retained += arena->stats.retained; 3384 astats->npurge += arena->stats.npurge; 3385 astats->nmadvise += arena->stats.nmadvise; 3386 astats->purged += arena->stats.purged; 3387 astats->metadata_mapped += arena->stats.metadata_mapped; 3388 astats->metadata_allocated += arena_metadata_allocated_get(arena); 3389 astats->allocated_large += arena->stats.allocated_large; 3390 astats->nmalloc_large += arena->stats.nmalloc_large; 3391 astats->ndalloc_large += arena->stats.ndalloc_large; 3392 astats->nrequests_large += arena->stats.nrequests_large; 3393 astats->allocated_huge += arena->stats.allocated_huge; 3394 astats->nmalloc_huge += arena->stats.nmalloc_huge; 3395 astats->ndalloc_huge += arena->stats.ndalloc_huge; 3396 3397 for (i = 0; i < nlclasses; i++) { 3398 lstats[i].nmalloc += arena->stats.lstats[i].nmalloc; 3399 lstats[i].ndalloc += arena->stats.lstats[i].ndalloc; 3400 lstats[i].nrequests += arena->stats.lstats[i].nrequests; 3401 lstats[i].curruns += arena->stats.lstats[i].curruns; 3402 } 3403 3404 for (i = 0; i < nhclasses; i++) { 3405 hstats[i].nmalloc += arena->stats.hstats[i].nmalloc; 3406 hstats[i].ndalloc += arena->stats.hstats[i].ndalloc; 3407 hstats[i].curhchunks += arena->stats.hstats[i].curhchunks; 3408 } 3409 malloc_mutex_unlock(tsdn, &arena->lock); 3410 3411 for (i = 0; i < NBINS; i++) { 3412 arena_bin_t *bin = &arena->bins[i]; 3413 3414 malloc_mutex_lock(tsdn, &bin->lock); 3415 bstats[i].nmalloc += bin->stats.nmalloc; 3416 bstats[i].ndalloc += bin->stats.ndalloc; 3417 bstats[i].nrequests += bin->stats.nrequests; 3418 bstats[i].curregs += bin->stats.curregs; 3419 if (config_tcache) { 3420 bstats[i].nfills += bin->stats.nfills; 3421 bstats[i].nflushes += bin->stats.nflushes; 3422 } 3423 bstats[i].nruns += bin->stats.nruns; 3424 bstats[i].reruns += bin->stats.reruns; 3425 bstats[i].curruns += bin->stats.curruns; 3426 malloc_mutex_unlock(tsdn, &bin->lock); 3427 } 3428} 3429 3430unsigned 3431arena_nthreads_get(arena_t *arena, bool internal) 3432{ 3433 3434 return (atomic_read_u(&arena->nthreads[internal])); 3435} 3436 3437void 3438arena_nthreads_inc(arena_t *arena, bool internal) 3439{ 3440 3441 atomic_add_u(&arena->nthreads[internal], 1); 3442} 3443 3444void 3445arena_nthreads_dec(arena_t *arena, bool internal) 3446{ 3447 3448 atomic_sub_u(&arena->nthreads[internal], 1); 3449} 3450 3451arena_t * 3452arena_new(tsdn_t *tsdn, unsigned ind) 3453{ 3454 arena_t *arena; 3455 unsigned i; 3456 3457 /* 3458 * Allocate arena, arena->lstats, and arena->hstats contiguously, mainly 3459 * because there is no way to clean up if base_alloc() OOMs. 3460 */ 3461 if (config_stats) { 3462 arena = (arena_t *)base_alloc(tsdn, 3463 CACHELINE_CEILING(sizeof(arena_t)) + 3464 QUANTUM_CEILING((nlclasses * sizeof(malloc_large_stats_t))) 3465 + (nhclasses * sizeof(malloc_huge_stats_t))); 3466 } else 3467 arena = (arena_t *)base_alloc(tsdn, sizeof(arena_t)); 3468 if (arena == NULL) 3469 return (NULL); 3470 3471 arena->ind = ind; 3472 arena->nthreads[0] = arena->nthreads[1] = 0; 3473 if (malloc_mutex_init(&arena->lock, "arena", WITNESS_RANK_ARENA)) 3474 return (NULL); 3475 3476 if (config_stats) { 3477 memset(&arena->stats, 0, sizeof(arena_stats_t)); 3478 arena->stats.lstats = (malloc_large_stats_t *)((uintptr_t)arena 3479 + CACHELINE_CEILING(sizeof(arena_t))); 3480 memset(arena->stats.lstats, 0, nlclasses * 3481 sizeof(malloc_large_stats_t)); 3482 arena->stats.hstats = (malloc_huge_stats_t *)((uintptr_t)arena 3483 + CACHELINE_CEILING(sizeof(arena_t)) + 3484 QUANTUM_CEILING(nlclasses * sizeof(malloc_large_stats_t))); 3485 memset(arena->stats.hstats, 0, nhclasses * 3486 sizeof(malloc_huge_stats_t)); 3487 if (config_tcache) 3488 ql_new(&arena->tcache_ql); 3489 } 3490 3491 if (config_prof) 3492 arena->prof_accumbytes = 0; 3493 3494 if (config_cache_oblivious) { 3495 /* 3496 * A nondeterministic seed based on the address of arena reduces 3497 * the likelihood of lockstep non-uniform cache index 3498 * utilization among identical concurrent processes, but at the 3499 * cost of test repeatability. For debug builds, instead use a 3500 * deterministic seed. 3501 */ 3502 arena->offset_state = config_debug ? ind : 3503 (uint64_t)(uintptr_t)arena; 3504 } 3505 3506 arena->dss_prec = chunk_dss_prec_get(); 3507 3508 ql_new(&arena->achunks); 3509 3510 arena->spare = NULL; 3511 3512 arena->lg_dirty_mult = arena_lg_dirty_mult_default_get(); 3513 arena->purging = false; 3514 arena->nactive = 0; 3515 arena->ndirty = 0; 3516 3517 for (i = 0; i < sizeof(arena->runs_avail) / sizeof(arena_run_heap_t); 3518 i++) 3519 arena_run_heap_new(&arena->runs_avail[i]); 3520 3521 qr_new(&arena->runs_dirty, rd_link); 3522 qr_new(&arena->chunks_cache, cc_link); 3523 3524 if (opt_purge == purge_mode_decay) 3525 arena_decay_init(arena, arena_decay_time_default_get()); 3526 3527 ql_new(&arena->huge); 3528 if (malloc_mutex_init(&arena->huge_mtx, "arena_huge", 3529 WITNESS_RANK_ARENA_HUGE)) 3530 return (NULL); 3531 3532 extent_tree_szad_new(&arena->chunks_szad_cached); 3533 extent_tree_ad_new(&arena->chunks_ad_cached); 3534 extent_tree_szad_new(&arena->chunks_szad_retained); 3535 extent_tree_ad_new(&arena->chunks_ad_retained); 3536 if (malloc_mutex_init(&arena->chunks_mtx, "arena_chunks", 3537 WITNESS_RANK_ARENA_CHUNKS)) 3538 return (NULL); 3539 ql_new(&arena->node_cache); 3540 if (malloc_mutex_init(&arena->node_cache_mtx, "arena_node_cache", 3541 WITNESS_RANK_ARENA_NODE_CACHE)) 3542 return (NULL); 3543 3544 arena->chunk_hooks = chunk_hooks_default; 3545 3546 /* Initialize bins. */ 3547 for (i = 0; i < NBINS; i++) { 3548 arena_bin_t *bin = &arena->bins[i]; 3549 if (malloc_mutex_init(&bin->lock, "arena_bin", 3550 WITNESS_RANK_ARENA_BIN)) 3551 return (NULL); 3552 bin->runcur = NULL; 3553 arena_run_heap_new(&bin->runs); 3554 if (config_stats) 3555 memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 3556 } 3557 3558 return (arena); 3559} 3560 3561/* 3562 * Calculate bin_info->run_size such that it meets the following constraints: 3563 * 3564 * *) bin_info->run_size <= arena_maxrun 3565 * *) bin_info->nregs <= RUN_MAXREGS 3566 * 3567 * bin_info->nregs and bin_info->reg0_offset are also calculated here, since 3568 * these settings are all interdependent. 3569 */ 3570static void 3571bin_info_run_size_calc(arena_bin_info_t *bin_info) 3572{ 3573 size_t pad_size; 3574 size_t try_run_size, perfect_run_size, actual_run_size; 3575 uint32_t try_nregs, perfect_nregs, actual_nregs; 3576 3577 /* 3578 * Determine redzone size based on minimum alignment and minimum 3579 * redzone size. Add padding to the end of the run if it is needed to 3580 * align the regions. The padding allows each redzone to be half the 3581 * minimum alignment; without the padding, each redzone would have to 3582 * be twice as large in order to maintain alignment. 3583 */ 3584 if (config_fill && unlikely(opt_redzone)) { 3585 size_t align_min = ZU(1) << (ffs_zu(bin_info->reg_size) - 1); 3586 if (align_min <= REDZONE_MINSIZE) { 3587 bin_info->redzone_size = REDZONE_MINSIZE; 3588 pad_size = 0; 3589 } else { 3590 bin_info->redzone_size = align_min >> 1; 3591 pad_size = bin_info->redzone_size; 3592 } 3593 } else { 3594 bin_info->redzone_size = 0; 3595 pad_size = 0; 3596 } 3597 bin_info->reg_interval = bin_info->reg_size + 3598 (bin_info->redzone_size << 1); 3599 3600 /* 3601 * Compute run size under ideal conditions (no redzones, no limit on run 3602 * size). 3603 */ 3604 try_run_size = PAGE; 3605 try_nregs = (uint32_t)(try_run_size / bin_info->reg_size); 3606 do { 3607 perfect_run_size = try_run_size; 3608 perfect_nregs = try_nregs; 3609 3610 try_run_size += PAGE; 3611 try_nregs = (uint32_t)(try_run_size / bin_info->reg_size); 3612 } while (perfect_run_size != perfect_nregs * bin_info->reg_size); 3613 assert(perfect_nregs <= RUN_MAXREGS); 3614 3615 actual_run_size = perfect_run_size; 3616 actual_nregs = (uint32_t)((actual_run_size - pad_size) / 3617 bin_info->reg_interval); 3618 3619 /* 3620 * Redzones can require enough padding that not even a single region can 3621 * fit within the number of pages that would normally be dedicated to a 3622 * run for this size class. Increase the run size until at least one 3623 * region fits. 3624 */ 3625 while (actual_nregs == 0) { 3626 assert(config_fill && unlikely(opt_redzone)); 3627 3628 actual_run_size += PAGE; 3629 actual_nregs = (uint32_t)((actual_run_size - pad_size) / 3630 bin_info->reg_interval); 3631 } 3632 3633 /* 3634 * Make sure that the run will fit within an arena chunk. 3635 */ 3636 while (actual_run_size > arena_maxrun) { 3637 actual_run_size -= PAGE; 3638 actual_nregs = (uint32_t)((actual_run_size - pad_size) / 3639 bin_info->reg_interval); 3640 } 3641 assert(actual_nregs > 0); 3642 assert(actual_run_size == s2u(actual_run_size)); 3643 3644 /* Copy final settings. */ 3645 bin_info->run_size = actual_run_size; 3646 bin_info->nregs = actual_nregs; 3647 bin_info->reg0_offset = (uint32_t)(actual_run_size - (actual_nregs * 3648 bin_info->reg_interval) - pad_size + bin_info->redzone_size); 3649 3650 assert(bin_info->reg0_offset - bin_info->redzone_size + (bin_info->nregs 3651 * bin_info->reg_interval) + pad_size == bin_info->run_size); 3652} 3653 3654static void 3655bin_info_init(void) 3656{ 3657 arena_bin_info_t *bin_info; 3658 3659#define BIN_INFO_INIT_bin_yes(index, size) \ 3660 bin_info = &arena_bin_info[index]; \ 3661 bin_info->reg_size = size; \ 3662 bin_info_run_size_calc(bin_info); \ 3663 bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); 3664#define BIN_INFO_INIT_bin_no(index, size) 3665#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup) \ 3666 BIN_INFO_INIT_bin_##bin(index, (ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta)) 3667 SIZE_CLASSES 3668#undef BIN_INFO_INIT_bin_yes 3669#undef BIN_INFO_INIT_bin_no 3670#undef SC 3671} 3672 3673void 3674arena_boot(void) 3675{ 3676 unsigned i; 3677 3678 arena_lg_dirty_mult_default_set(opt_lg_dirty_mult); 3679 arena_decay_time_default_set(opt_decay_time); 3680 3681 /* 3682 * Compute the header size such that it is large enough to contain the 3683 * page map. The page map is biased to omit entries for the header 3684 * itself, so some iteration is necessary to compute the map bias. 3685 * 3686 * 1) Compute safe header_size and map_bias values that include enough 3687 * space for an unbiased page map. 3688 * 2) Refine map_bias based on (1) to omit the header pages in the page 3689 * map. The resulting map_bias may be one too small. 3690 * 3) Refine map_bias based on (2). The result will be >= the result 3691 * from (2), and will always be correct. 3692 */ 3693 map_bias = 0; 3694 for (i = 0; i < 3; i++) { 3695 size_t header_size = offsetof(arena_chunk_t, map_bits) + 3696 ((sizeof(arena_chunk_map_bits_t) + 3697 sizeof(arena_chunk_map_misc_t)) * (chunk_npages-map_bias)); 3698 map_bias = (header_size + PAGE_MASK) >> LG_PAGE; 3699 } 3700 assert(map_bias > 0); 3701 3702 map_misc_offset = offsetof(arena_chunk_t, map_bits) + 3703 sizeof(arena_chunk_map_bits_t) * (chunk_npages-map_bias); 3704 3705 arena_maxrun = chunksize - (map_bias << LG_PAGE); 3706 assert(arena_maxrun > 0); 3707 large_maxclass = index2size(size2index(chunksize)-1); 3708 if (large_maxclass > arena_maxrun) { 3709 /* 3710 * For small chunk sizes it's possible for there to be fewer 3711 * non-header pages available than are necessary to serve the 3712 * size classes just below chunksize. 3713 */ 3714 large_maxclass = arena_maxrun; 3715 } 3716 assert(large_maxclass > 0); 3717 nlclasses = size2index(large_maxclass) - size2index(SMALL_MAXCLASS); 3718 nhclasses = NSIZES - nlclasses - NBINS; 3719 3720 bin_info_init(); 3721} 3722 3723void 3724arena_prefork0(tsdn_t *tsdn, arena_t *arena) 3725{ 3726 3727 malloc_mutex_prefork(tsdn, &arena->lock); 3728} 3729 3730void 3731arena_prefork1(tsdn_t *tsdn, arena_t *arena) 3732{ 3733 3734 malloc_mutex_prefork(tsdn, &arena->chunks_mtx); 3735} 3736 3737void 3738arena_prefork2(tsdn_t *tsdn, arena_t *arena) 3739{ 3740 3741 malloc_mutex_prefork(tsdn, &arena->node_cache_mtx); 3742} 3743 3744void 3745arena_prefork3(tsdn_t *tsdn, arena_t *arena) 3746{ 3747 unsigned i; 3748 3749 for (i = 0; i < NBINS; i++) 3750 malloc_mutex_prefork(tsdn, &arena->bins[i].lock); 3751 malloc_mutex_prefork(tsdn, &arena->huge_mtx); 3752} 3753 3754void 3755arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) 3756{ 3757 unsigned i; 3758 3759 malloc_mutex_postfork_parent(tsdn, &arena->huge_mtx); 3760 for (i = 0; i < NBINS; i++) 3761 malloc_mutex_postfork_parent(tsdn, &arena->bins[i].lock); 3762 malloc_mutex_postfork_parent(tsdn, &arena->node_cache_mtx); 3763 malloc_mutex_postfork_parent(tsdn, &arena->chunks_mtx); 3764 malloc_mutex_postfork_parent(tsdn, &arena->lock); 3765} 3766 3767void 3768arena_postfork_child(tsdn_t *tsdn, arena_t *arena) 3769{ 3770 unsigned i; 3771 3772 malloc_mutex_postfork_child(tsdn, &arena->huge_mtx); 3773 for (i = 0; i < NBINS; i++) 3774 malloc_mutex_postfork_child(tsdn, &arena->bins[i].lock); 3775 malloc_mutex_postfork_child(tsdn, &arena->node_cache_mtx); 3776 malloc_mutex_postfork_child(tsdn, &arena->chunks_mtx); 3777 malloc_mutex_postfork_child(tsdn, &arena->lock); 3778} 3779