arena.c revision f9ff60346d7c25ad653ea062e496a5d0864233b2
1#define JEMALLOC_ARENA_C_ 2#include "jemalloc/internal/jemalloc_internal.h" 3 4/******************************************************************************/ 5/* Data. */ 6 7ssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; 8arena_bin_info_t arena_bin_info[NBINS]; 9 10JEMALLOC_ALIGNED(CACHELINE) 11const uint8_t small_size2bin[] = { 12#define S2B_8(i) i, 13#define S2B_16(i) S2B_8(i) S2B_8(i) 14#define S2B_32(i) S2B_16(i) S2B_16(i) 15#define S2B_64(i) S2B_32(i) S2B_32(i) 16#define S2B_128(i) S2B_64(i) S2B_64(i) 17#define S2B_256(i) S2B_128(i) S2B_128(i) 18#define S2B_512(i) S2B_256(i) S2B_256(i) 19#define S2B_1024(i) S2B_512(i) S2B_512(i) 20#define S2B_2048(i) S2B_1024(i) S2B_1024(i) 21#define S2B_4096(i) S2B_2048(i) S2B_2048(i) 22#define S2B_8192(i) S2B_4096(i) S2B_4096(i) 23#define SIZE_CLASS(bin, delta, size) \ 24 S2B_##delta(bin) 25 SIZE_CLASSES 26#undef S2B_8 27#undef S2B_16 28#undef S2B_32 29#undef S2B_64 30#undef S2B_128 31#undef S2B_256 32#undef S2B_512 33#undef S2B_1024 34#undef S2B_2048 35#undef S2B_4096 36#undef S2B_8192 37#undef SIZE_CLASS 38}; 39 40/******************************************************************************/ 41/* 42 * Function prototypes for static functions that are referenced prior to 43 * definition. 44 */ 45 46static void arena_purge(arena_t *arena, bool all); 47static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, 48 bool cleaned); 49static void arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, 50 arena_run_t *run, arena_bin_t *bin); 51static void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, 52 arena_run_t *run, arena_bin_t *bin); 53 54/******************************************************************************/ 55 56JEMALLOC_INLINE_C size_t 57arena_mapelm_to_pageind(arena_chunk_map_t *mapelm) 58{ 59 uintptr_t map_offset = 60 CHUNK_ADDR2OFFSET(mapelm) - offsetof(arena_chunk_t, map); 61 62 return ((map_offset / sizeof(arena_chunk_map_t)) + map_bias); 63} 64 65JEMALLOC_INLINE_C size_t 66arena_mapelm_to_bits(arena_chunk_map_t *mapelm) 67{ 68 69 return (mapelm->bits); 70} 71 72static inline int 73arena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) 74{ 75 uintptr_t a_mapelm = (uintptr_t)a; 76 uintptr_t b_mapelm = (uintptr_t)b; 77 78 assert(a != NULL); 79 assert(b != NULL); 80 81 return ((a_mapelm > b_mapelm) - (a_mapelm < b_mapelm)); 82} 83 84/* Generate red-black tree functions. */ 85rb_gen(static UNUSED, arena_run_tree_, arena_run_tree_t, arena_chunk_map_t, 86 u.rb_link, arena_run_comp) 87 88static inline int 89arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) 90{ 91 int ret; 92 size_t a_size; 93 size_t b_size = arena_mapelm_to_bits(b) & ~PAGE_MASK; 94 uintptr_t a_mapelm = (uintptr_t)a; 95 uintptr_t b_mapelm = (uintptr_t)b; 96 97 if (a_mapelm & CHUNK_MAP_KEY) 98 a_size = a_mapelm & ~PAGE_MASK; 99 else 100 a_size = arena_mapelm_to_bits(a) & ~PAGE_MASK; 101 102 ret = (a_size > b_size) - (a_size < b_size); 103 if (ret == 0 && (!(a_mapelm & CHUNK_MAP_KEY))) 104 ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm); 105 106 return (ret); 107} 108 109/* Generate red-black tree functions. */ 110rb_gen(static UNUSED, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t, 111 u.rb_link, arena_avail_comp) 112 113static inline int 114arena_chunk_dirty_comp(arena_chunk_t *a, arena_chunk_t *b) 115{ 116 117 assert(a != NULL); 118 assert(b != NULL); 119 120 /* 121 * Short-circuit for self comparison. The following comparison code 122 * would come to the same result, but at the cost of executing the slow 123 * path. 124 */ 125 if (a == b) 126 return (0); 127 128 /* 129 * Order such that chunks with higher fragmentation are "less than" 130 * those with lower fragmentation -- purging order is from "least" to 131 * "greatest". Fragmentation is measured as: 132 * 133 * mean current avail run size 134 * -------------------------------- 135 * mean defragmented avail run size 136 * 137 * navail 138 * ----------- 139 * nruns_avail nruns_avail-nruns_adjac 140 * = ========================= = ----------------------- 141 * navail nruns_avail 142 * ----------------------- 143 * nruns_avail-nruns_adjac 144 * 145 * The following code multiplies away the denominator prior to 146 * comparison, in order to avoid division. 147 * 148 */ 149 { 150 size_t a_val = (a->nruns_avail - a->nruns_adjac) * 151 b->nruns_avail; 152 size_t b_val = (b->nruns_avail - b->nruns_adjac) * 153 a->nruns_avail; 154 155 if (a_val < b_val) 156 return (1); 157 if (a_val > b_val) 158 return (-1); 159 } 160 /* 161 * Break ties by chunk address. For fragmented chunks, report lower 162 * addresses as "lower", so that fragmentation reduction happens first 163 * at lower addresses. However, use the opposite ordering for 164 * unfragmented chunks, in order to increase the chances of 165 * re-allocating dirty runs. 166 */ 167 { 168 uintptr_t a_chunk = (uintptr_t)a; 169 uintptr_t b_chunk = (uintptr_t)b; 170 int ret = ((a_chunk > b_chunk) - (a_chunk < b_chunk)); 171 if (a->nruns_adjac == 0) { 172 assert(b->nruns_adjac == 0); 173 ret = -ret; 174 } 175 return (ret); 176 } 177} 178 179/* Generate red-black tree functions. */ 180rb_gen(static UNUSED, arena_chunk_dirty_, arena_chunk_tree_t, arena_chunk_t, 181 dirty_link, arena_chunk_dirty_comp) 182 183static inline bool 184arena_avail_adjac_pred(arena_chunk_t *chunk, size_t pageind) 185{ 186 bool ret; 187 188 if (pageind-1 < map_bias) 189 ret = false; 190 else { 191 ret = (arena_mapbits_allocated_get(chunk, pageind-1) == 0); 192 assert(ret == false || arena_mapbits_dirty_get(chunk, 193 pageind-1) != arena_mapbits_dirty_get(chunk, pageind)); 194 } 195 return (ret); 196} 197 198static inline bool 199arena_avail_adjac_succ(arena_chunk_t *chunk, size_t pageind, size_t npages) 200{ 201 bool ret; 202 203 if (pageind+npages == chunk_npages) 204 ret = false; 205 else { 206 assert(pageind+npages < chunk_npages); 207 ret = (arena_mapbits_allocated_get(chunk, pageind+npages) == 0); 208 assert(ret == false || arena_mapbits_dirty_get(chunk, pageind) 209 != arena_mapbits_dirty_get(chunk, pageind+npages)); 210 } 211 return (ret); 212} 213 214static inline bool 215arena_avail_adjac(arena_chunk_t *chunk, size_t pageind, size_t npages) 216{ 217 218 return (arena_avail_adjac_pred(chunk, pageind) || 219 arena_avail_adjac_succ(chunk, pageind, npages)); 220} 221 222static void 223arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 224 size_t npages, bool maybe_adjac_pred, bool maybe_adjac_succ) 225{ 226 227 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 228 LG_PAGE)); 229 230 /* 231 * chunks_dirty is keyed by nruns_{avail,adjac}, so the chunk must be 232 * removed and reinserted even if the run to be inserted is clean. 233 */ 234 if (chunk->ndirty != 0) 235 arena_chunk_dirty_remove(&arena->chunks_dirty, chunk); 236 237 if (maybe_adjac_pred && arena_avail_adjac_pred(chunk, pageind)) 238 chunk->nruns_adjac++; 239 if (maybe_adjac_succ && arena_avail_adjac_succ(chunk, pageind, npages)) 240 chunk->nruns_adjac++; 241 chunk->nruns_avail++; 242 assert(chunk->nruns_avail > chunk->nruns_adjac); 243 244 if (arena_mapbits_dirty_get(chunk, pageind) != 0) { 245 arena->ndirty += npages; 246 chunk->ndirty += npages; 247 } 248 if (chunk->ndirty != 0) 249 arena_chunk_dirty_insert(&arena->chunks_dirty, chunk); 250 251 arena_avail_tree_insert(&arena->runs_avail, arena_mapp_get(chunk, 252 pageind)); 253} 254 255static void 256arena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 257 size_t npages, bool maybe_adjac_pred, bool maybe_adjac_succ) 258{ 259 260 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 261 LG_PAGE)); 262 263 /* 264 * chunks_dirty is keyed by nruns_{avail,adjac}, so the chunk must be 265 * removed and reinserted even if the run to be removed is clean. 266 */ 267 if (chunk->ndirty != 0) 268 arena_chunk_dirty_remove(&arena->chunks_dirty, chunk); 269 270 if (maybe_adjac_pred && arena_avail_adjac_pred(chunk, pageind)) 271 chunk->nruns_adjac--; 272 if (maybe_adjac_succ && arena_avail_adjac_succ(chunk, pageind, npages)) 273 chunk->nruns_adjac--; 274 chunk->nruns_avail--; 275 assert(chunk->nruns_avail > chunk->nruns_adjac || (chunk->nruns_avail 276 == 0 && chunk->nruns_adjac == 0)); 277 278 if (arena_mapbits_dirty_get(chunk, pageind) != 0) { 279 arena->ndirty -= npages; 280 chunk->ndirty -= npages; 281 } 282 if (chunk->ndirty != 0) 283 arena_chunk_dirty_insert(&arena->chunks_dirty, chunk); 284 285 arena_avail_tree_remove(&arena->runs_avail, arena_mapp_get(chunk, 286 pageind)); 287} 288 289static inline void * 290arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) 291{ 292 void *ret; 293 unsigned regind; 294 bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 295 (uintptr_t)bin_info->bitmap_offset); 296 297 assert(run->nfree > 0); 298 assert(bitmap_full(bitmap, &bin_info->bitmap_info) == false); 299 300 regind = bitmap_sfu(bitmap, &bin_info->bitmap_info); 301 ret = (void *)((uintptr_t)run + (uintptr_t)bin_info->reg0_offset + 302 (uintptr_t)(bin_info->reg_interval * regind)); 303 run->nfree--; 304 if (regind == run->nextind) 305 run->nextind++; 306 assert(regind < run->nextind); 307 return (ret); 308} 309 310static inline void 311arena_run_reg_dalloc(arena_run_t *run, void *ptr) 312{ 313 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 314 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 315 size_t mapbits = arena_mapbits_get(chunk, pageind); 316 size_t binind = arena_ptr_small_binind_get(ptr, mapbits); 317 arena_bin_info_t *bin_info = &arena_bin_info[binind]; 318 unsigned regind = arena_run_regind(run, bin_info, ptr); 319 bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 320 (uintptr_t)bin_info->bitmap_offset); 321 322 assert(run->nfree < bin_info->nregs); 323 /* Freeing an interior pointer can cause assertion failure. */ 324 assert(((uintptr_t)ptr - ((uintptr_t)run + 325 (uintptr_t)bin_info->reg0_offset)) % 326 (uintptr_t)bin_info->reg_interval == 0); 327 assert((uintptr_t)ptr >= (uintptr_t)run + 328 (uintptr_t)bin_info->reg0_offset); 329 /* Freeing an unallocated pointer can cause assertion failure. */ 330 assert(bitmap_get(bitmap, &bin_info->bitmap_info, regind)); 331 332 bitmap_unset(bitmap, &bin_info->bitmap_info, regind); 333 run->nfree++; 334} 335 336static inline void 337arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages) 338{ 339 340 VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + (run_ind << 341 LG_PAGE)), (npages << LG_PAGE)); 342 memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0, 343 (npages << LG_PAGE)); 344} 345 346static inline void 347arena_run_page_mark_zeroed(arena_chunk_t *chunk, size_t run_ind) 348{ 349 350 VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind << 351 LG_PAGE)), PAGE); 352} 353 354static inline void 355arena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) 356{ 357 size_t i; 358 UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE)); 359 360 arena_run_page_mark_zeroed(chunk, run_ind); 361 for (i = 0; i < PAGE / sizeof(size_t); i++) 362 assert(p[i] == 0); 363} 364 365static void 366arena_cactive_update(arena_t *arena, size_t add_pages, size_t sub_pages) 367{ 368 369 if (config_stats) { 370 ssize_t cactive_diff = CHUNK_CEILING((arena->nactive + 371 add_pages) << LG_PAGE) - CHUNK_CEILING((arena->nactive - 372 sub_pages) << LG_PAGE); 373 if (cactive_diff != 0) 374 stats_cactive_add(cactive_diff); 375 } 376} 377 378static void 379arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, 380 size_t flag_dirty, size_t need_pages) 381{ 382 size_t total_pages, rem_pages; 383 384 total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >> 385 LG_PAGE; 386 assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) == 387 flag_dirty); 388 assert(need_pages <= total_pages); 389 rem_pages = total_pages - need_pages; 390 391 arena_avail_remove(arena, chunk, run_ind, total_pages, true, true); 392 arena_cactive_update(arena, need_pages, 0); 393 arena->nactive += need_pages; 394 395 /* Keep track of trailing unused pages for later use. */ 396 if (rem_pages > 0) { 397 if (flag_dirty != 0) { 398 arena_mapbits_unallocated_set(chunk, 399 run_ind+need_pages, (rem_pages << LG_PAGE), 400 flag_dirty); 401 arena_mapbits_unallocated_set(chunk, 402 run_ind+total_pages-1, (rem_pages << LG_PAGE), 403 flag_dirty); 404 } else { 405 arena_mapbits_unallocated_set(chunk, run_ind+need_pages, 406 (rem_pages << LG_PAGE), 407 arena_mapbits_unzeroed_get(chunk, 408 run_ind+need_pages)); 409 arena_mapbits_unallocated_set(chunk, 410 run_ind+total_pages-1, (rem_pages << LG_PAGE), 411 arena_mapbits_unzeroed_get(chunk, 412 run_ind+total_pages-1)); 413 } 414 arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages, 415 false, true); 416 } 417} 418 419static void 420arena_run_split_large_helper(arena_t *arena, arena_run_t *run, size_t size, 421 bool remove, bool zero) 422{ 423 arena_chunk_t *chunk; 424 size_t flag_dirty, run_ind, need_pages, i; 425 426 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 427 run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 428 flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); 429 need_pages = (size >> LG_PAGE); 430 assert(need_pages > 0); 431 432 if (remove) { 433 arena_run_split_remove(arena, chunk, run_ind, flag_dirty, 434 need_pages); 435 } 436 437 if (zero) { 438 if (flag_dirty == 0) { 439 /* 440 * The run is clean, so some pages may be zeroed (i.e. 441 * never before touched). 442 */ 443 for (i = 0; i < need_pages; i++) { 444 if (arena_mapbits_unzeroed_get(chunk, run_ind+i) 445 != 0) 446 arena_run_zero(chunk, run_ind+i, 1); 447 else if (config_debug) { 448 arena_run_page_validate_zeroed(chunk, 449 run_ind+i); 450 } else { 451 arena_run_page_mark_zeroed(chunk, 452 run_ind+i); 453 } 454 } 455 } else { 456 /* The run is dirty, so all pages must be zeroed. */ 457 arena_run_zero(chunk, run_ind, need_pages); 458 } 459 } else { 460 VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + 461 (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); 462 } 463 464 /* 465 * Set the last element first, in case the run only contains one page 466 * (i.e. both statements set the same element). 467 */ 468 arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0, flag_dirty); 469 arena_mapbits_large_set(chunk, run_ind, size, flag_dirty); 470} 471 472static void 473arena_run_split_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) 474{ 475 476 arena_run_split_large_helper(arena, run, size, true, zero); 477} 478 479static void 480arena_run_init_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) 481{ 482 483 arena_run_split_large_helper(arena, run, size, false, zero); 484} 485 486static void 487arena_run_split_small(arena_t *arena, arena_run_t *run, size_t size, 488 size_t binind) 489{ 490 arena_chunk_t *chunk; 491 size_t flag_dirty, run_ind, need_pages, i; 492 493 assert(binind != BININD_INVALID); 494 495 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 496 run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 497 flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); 498 need_pages = (size >> LG_PAGE); 499 assert(need_pages > 0); 500 501 arena_run_split_remove(arena, chunk, run_ind, flag_dirty, need_pages); 502 503 /* 504 * Propagate the dirty and unzeroed flags to the allocated small run, 505 * so that arena_dalloc_bin_run() has the ability to conditionally trim 506 * clean pages. 507 */ 508 arena_mapbits_small_set(chunk, run_ind, 0, binind, flag_dirty); 509 /* 510 * The first page will always be dirtied during small run 511 * initialization, so a validation failure here would not actually 512 * cause an observable failure. 513 */ 514 if (config_debug && flag_dirty == 0 && arena_mapbits_unzeroed_get(chunk, 515 run_ind) == 0) 516 arena_run_page_validate_zeroed(chunk, run_ind); 517 for (i = 1; i < need_pages - 1; i++) { 518 arena_mapbits_small_set(chunk, run_ind+i, i, binind, 0); 519 if (config_debug && flag_dirty == 0 && 520 arena_mapbits_unzeroed_get(chunk, run_ind+i) == 0) 521 arena_run_page_validate_zeroed(chunk, run_ind+i); 522 } 523 arena_mapbits_small_set(chunk, run_ind+need_pages-1, need_pages-1, 524 binind, flag_dirty); 525 if (config_debug && flag_dirty == 0 && arena_mapbits_unzeroed_get(chunk, 526 run_ind+need_pages-1) == 0) 527 arena_run_page_validate_zeroed(chunk, run_ind+need_pages-1); 528 VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + 529 (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); 530} 531 532static arena_chunk_t * 533arena_chunk_init_spare(arena_t *arena) 534{ 535 arena_chunk_t *chunk; 536 537 assert(arena->spare != NULL); 538 539 chunk = arena->spare; 540 arena->spare = NULL; 541 542 assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); 543 assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); 544 assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == 545 arena_maxclass); 546 assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == 547 arena_maxclass); 548 assert(arena_mapbits_dirty_get(chunk, map_bias) == 549 arena_mapbits_dirty_get(chunk, chunk_npages-1)); 550 551 return (chunk); 552} 553 554static arena_chunk_t * 555arena_chunk_init_hard(arena_t *arena) 556{ 557 arena_chunk_t *chunk; 558 bool zero; 559 size_t unzeroed, i; 560 561 assert(arena->spare == NULL); 562 563 zero = false; 564 malloc_mutex_unlock(&arena->lock); 565 chunk = (arena_chunk_t *)chunk_alloc(chunksize, chunksize, false, 566 &zero, arena->dss_prec); 567 malloc_mutex_lock(&arena->lock); 568 if (chunk == NULL) 569 return (NULL); 570 if (config_stats) 571 arena->stats.mapped += chunksize; 572 573 chunk->arena = arena; 574 575 /* 576 * Claim that no pages are in use, since the header is merely overhead. 577 */ 578 chunk->ndirty = 0; 579 580 chunk->nruns_avail = 0; 581 chunk->nruns_adjac = 0; 582 583 /* 584 * Initialize the map to contain one maximal free untouched run. Mark 585 * the pages as zeroed iff chunk_alloc() returned a zeroed chunk. 586 */ 587 unzeroed = zero ? 0 : CHUNK_MAP_UNZEROED; 588 arena_mapbits_unallocated_set(chunk, map_bias, arena_maxclass, 589 unzeroed); 590 /* 591 * There is no need to initialize the internal page map entries unless 592 * the chunk is not zeroed. 593 */ 594 if (zero == false) { 595 VALGRIND_MAKE_MEM_UNDEFINED((void *)arena_mapp_get(chunk, 596 map_bias+1), (size_t)((uintptr_t) arena_mapp_get(chunk, 597 chunk_npages-1) - (uintptr_t)arena_mapp_get(chunk, 598 map_bias+1))); 599 for (i = map_bias+1; i < chunk_npages-1; i++) 600 arena_mapbits_unzeroed_set(chunk, i, unzeroed); 601 } else { 602 VALGRIND_MAKE_MEM_DEFINED((void *)arena_mapp_get(chunk, 603 map_bias+1), (size_t)((uintptr_t) arena_mapp_get(chunk, 604 chunk_npages-1) - (uintptr_t)arena_mapp_get(chunk, 605 map_bias+1))); 606 if (config_debug) { 607 for (i = map_bias+1; i < chunk_npages-1; i++) { 608 assert(arena_mapbits_unzeroed_get(chunk, i) == 609 unzeroed); 610 } 611 } 612 } 613 arena_mapbits_unallocated_set(chunk, chunk_npages-1, arena_maxclass, 614 unzeroed); 615 616 return (chunk); 617} 618 619static arena_chunk_t * 620arena_chunk_alloc(arena_t *arena) 621{ 622 arena_chunk_t *chunk; 623 624 if (arena->spare != NULL) 625 chunk = arena_chunk_init_spare(arena); 626 else { 627 chunk = arena_chunk_init_hard(arena); 628 if (chunk == NULL) 629 return (NULL); 630 } 631 632 /* Insert the run into the runs_avail tree. */ 633 arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias, 634 false, false); 635 636 return (chunk); 637} 638 639static void 640arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk) 641{ 642 assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); 643 assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); 644 assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == 645 arena_maxclass); 646 assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == 647 arena_maxclass); 648 assert(arena_mapbits_dirty_get(chunk, map_bias) == 649 arena_mapbits_dirty_get(chunk, chunk_npages-1)); 650 651 /* 652 * Remove run from the runs_avail tree, so that the arena does not use 653 * it. 654 */ 655 arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias, 656 false, false); 657 658 if (arena->spare != NULL) { 659 arena_chunk_t *spare = arena->spare; 660 661 arena->spare = chunk; 662 malloc_mutex_unlock(&arena->lock); 663 chunk_dealloc((void *)spare, chunksize, true); 664 malloc_mutex_lock(&arena->lock); 665 if (config_stats) 666 arena->stats.mapped -= chunksize; 667 } else 668 arena->spare = chunk; 669} 670 671static arena_run_t * 672arena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero) 673{ 674 arena_run_t *run; 675 arena_chunk_map_t *mapelm; 676 arena_chunk_map_t *key; 677 678 key = (arena_chunk_map_t *)(size | CHUNK_MAP_KEY); 679 mapelm = arena_avail_tree_nsearch(&arena->runs_avail, key); 680 if (mapelm != NULL) { 681 arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); 682 size_t pageind = arena_mapelm_to_pageind(mapelm); 683 684 run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << 685 LG_PAGE)); 686 arena_run_split_large(arena, run, size, zero); 687 return (run); 688 } 689 690 return (NULL); 691} 692 693static arena_run_t * 694arena_run_alloc_large(arena_t *arena, size_t size, bool zero) 695{ 696 arena_chunk_t *chunk; 697 arena_run_t *run; 698 699 assert(size <= arena_maxclass); 700 assert((size & PAGE_MASK) == 0); 701 702 /* Search the arena's chunks for the lowest best fit. */ 703 run = arena_run_alloc_large_helper(arena, size, zero); 704 if (run != NULL) 705 return (run); 706 707 /* 708 * No usable runs. Create a new chunk from which to allocate the run. 709 */ 710 chunk = arena_chunk_alloc(arena); 711 if (chunk != NULL) { 712 run = (arena_run_t *)((uintptr_t)chunk + (map_bias << LG_PAGE)); 713 arena_run_split_large(arena, run, size, zero); 714 return (run); 715 } 716 717 /* 718 * arena_chunk_alloc() failed, but another thread may have made 719 * sufficient memory available while this one dropped arena->lock in 720 * arena_chunk_alloc(), so search one more time. 721 */ 722 return (arena_run_alloc_large_helper(arena, size, zero)); 723} 724 725static arena_run_t * 726arena_run_alloc_small_helper(arena_t *arena, size_t size, size_t binind) 727{ 728 arena_run_t *run; 729 arena_chunk_map_t *mapelm; 730 arena_chunk_map_t *key; 731 732 key = (arena_chunk_map_t *)(size | CHUNK_MAP_KEY); 733 mapelm = arena_avail_tree_nsearch(&arena->runs_avail, key); 734 if (mapelm != NULL) { 735 arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); 736 size_t pageind = arena_mapelm_to_pageind(mapelm); 737 738 run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << 739 LG_PAGE)); 740 arena_run_split_small(arena, run, size, binind); 741 return (run); 742 } 743 744 return (NULL); 745} 746 747static arena_run_t * 748arena_run_alloc_small(arena_t *arena, size_t size, size_t binind) 749{ 750 arena_chunk_t *chunk; 751 arena_run_t *run; 752 753 assert(size <= arena_maxclass); 754 assert((size & PAGE_MASK) == 0); 755 assert(binind != BININD_INVALID); 756 757 /* Search the arena's chunks for the lowest best fit. */ 758 run = arena_run_alloc_small_helper(arena, size, binind); 759 if (run != NULL) 760 return (run); 761 762 /* 763 * No usable runs. Create a new chunk from which to allocate the run. 764 */ 765 chunk = arena_chunk_alloc(arena); 766 if (chunk != NULL) { 767 run = (arena_run_t *)((uintptr_t)chunk + (map_bias << LG_PAGE)); 768 arena_run_split_small(arena, run, size, binind); 769 return (run); 770 } 771 772 /* 773 * arena_chunk_alloc() failed, but another thread may have made 774 * sufficient memory available while this one dropped arena->lock in 775 * arena_chunk_alloc(), so search one more time. 776 */ 777 return (arena_run_alloc_small_helper(arena, size, binind)); 778} 779 780static inline void 781arena_maybe_purge(arena_t *arena) 782{ 783 size_t npurgeable, threshold; 784 785 /* Don't purge if the option is disabled. */ 786 if (opt_lg_dirty_mult < 0) 787 return; 788 /* Don't purge if all dirty pages are already being purged. */ 789 if (arena->ndirty <= arena->npurgatory) 790 return; 791 npurgeable = arena->ndirty - arena->npurgatory; 792 threshold = (arena->nactive >> opt_lg_dirty_mult); 793 /* 794 * Don't purge unless the number of purgeable pages exceeds the 795 * threshold. 796 */ 797 if (npurgeable <= threshold) 798 return; 799 800 arena_purge(arena, false); 801} 802 803static arena_chunk_t * 804chunks_dirty_iter_cb(arena_chunk_tree_t *tree, arena_chunk_t *chunk, void *arg) 805{ 806 size_t *ndirty = (size_t *)arg; 807 808 assert(chunk->ndirty != 0); 809 *ndirty += chunk->ndirty; 810 return (NULL); 811} 812 813static size_t 814arena_compute_npurgatory(arena_t *arena, bool all) 815{ 816 size_t npurgatory, npurgeable; 817 818 /* 819 * Compute the minimum number of pages that this thread should try to 820 * purge. 821 */ 822 npurgeable = arena->ndirty - arena->npurgatory; 823 824 if (all == false) { 825 size_t threshold = (arena->nactive >> opt_lg_dirty_mult); 826 827 npurgatory = npurgeable - threshold; 828 } else 829 npurgatory = npurgeable; 830 831 return (npurgatory); 832} 833 834static void 835arena_chunk_stash_dirty(arena_t *arena, arena_chunk_t *chunk, bool all, 836 arena_chunk_mapelms_t *mapelms) 837{ 838 size_t pageind, npages; 839 840 /* 841 * Temporarily allocate free dirty runs within chunk. If all is false, 842 * only operate on dirty runs that are fragments; otherwise operate on 843 * all dirty runs. 844 */ 845 for (pageind = map_bias; pageind < chunk_npages; pageind += npages) { 846 arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); 847 if (arena_mapbits_allocated_get(chunk, pageind) == 0) { 848 size_t run_size = 849 arena_mapbits_unallocated_size_get(chunk, pageind); 850 851 npages = run_size >> LG_PAGE; 852 assert(pageind + npages <= chunk_npages); 853 assert(arena_mapbits_dirty_get(chunk, pageind) == 854 arena_mapbits_dirty_get(chunk, pageind+npages-1)); 855 856 if (arena_mapbits_dirty_get(chunk, pageind) != 0 && 857 (all || arena_avail_adjac(chunk, pageind, 858 npages))) { 859 arena_run_t *run = (arena_run_t *)((uintptr_t) 860 chunk + (uintptr_t)(pageind << LG_PAGE)); 861 862 arena_run_split_large(arena, run, run_size, 863 false); 864 /* Append to list for later processing. */ 865 ql_elm_new(mapelm, u.ql_link); 866 ql_tail_insert(mapelms, mapelm, u.ql_link); 867 } 868 } else { 869 /* Skip run. */ 870 if (arena_mapbits_large_get(chunk, pageind) != 0) { 871 npages = arena_mapbits_large_size_get(chunk, 872 pageind) >> LG_PAGE; 873 } else { 874 size_t binind; 875 arena_bin_info_t *bin_info; 876 arena_run_t *run = (arena_run_t *)((uintptr_t) 877 chunk + (uintptr_t)(pageind << LG_PAGE)); 878 879 assert(arena_mapbits_small_runind_get(chunk, 880 pageind) == 0); 881 binind = arena_bin_index(arena, run->bin); 882 bin_info = &arena_bin_info[binind]; 883 npages = bin_info->run_size >> LG_PAGE; 884 } 885 } 886 } 887 assert(pageind == chunk_npages); 888 assert(chunk->ndirty == 0 || all == false); 889 assert(chunk->nruns_adjac == 0); 890} 891 892static size_t 893arena_chunk_purge_stashed(arena_t *arena, arena_chunk_t *chunk, 894 arena_chunk_mapelms_t *mapelms) 895{ 896 size_t npurged, pageind, npages, nmadvise; 897 arena_chunk_map_t *mapelm; 898 899 malloc_mutex_unlock(&arena->lock); 900 if (config_stats) 901 nmadvise = 0; 902 npurged = 0; 903 ql_foreach(mapelm, mapelms, u.ql_link) { 904 bool unzeroed; 905 size_t flag_unzeroed, i; 906 907 pageind = arena_mapelm_to_pageind(mapelm); 908 npages = arena_mapbits_large_size_get(chunk, pageind) >> 909 LG_PAGE; 910 assert(pageind + npages <= chunk_npages); 911 unzeroed = pages_purge((void *)((uintptr_t)chunk + (pageind << 912 LG_PAGE)), (npages << LG_PAGE)); 913 flag_unzeroed = unzeroed ? CHUNK_MAP_UNZEROED : 0; 914 /* 915 * Set the unzeroed flag for all pages, now that pages_purge() 916 * has returned whether the pages were zeroed as a side effect 917 * of purging. This chunk map modification is safe even though 918 * the arena mutex isn't currently owned by this thread, 919 * because the run is marked as allocated, thus protecting it 920 * from being modified by any other thread. As long as these 921 * writes don't perturb the first and last elements' 922 * CHUNK_MAP_ALLOCATED bits, behavior is well defined. 923 */ 924 for (i = 0; i < npages; i++) { 925 arena_mapbits_unzeroed_set(chunk, pageind+i, 926 flag_unzeroed); 927 } 928 npurged += npages; 929 if (config_stats) 930 nmadvise++; 931 } 932 malloc_mutex_lock(&arena->lock); 933 if (config_stats) 934 arena->stats.nmadvise += nmadvise; 935 936 return (npurged); 937} 938 939static void 940arena_chunk_unstash_purged(arena_t *arena, arena_chunk_t *chunk, 941 arena_chunk_mapelms_t *mapelms) 942{ 943 arena_chunk_map_t *mapelm; 944 size_t pageind; 945 946 /* Deallocate runs. */ 947 for (mapelm = ql_first(mapelms); mapelm != NULL; 948 mapelm = ql_first(mapelms)) { 949 arena_run_t *run; 950 951 pageind = arena_mapelm_to_pageind(mapelm); 952 run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)(pageind << 953 LG_PAGE)); 954 ql_remove(mapelms, mapelm, u.ql_link); 955 arena_run_dalloc(arena, run, false, true); 956 } 957} 958 959static inline size_t 960arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk, bool all) 961{ 962 size_t npurged; 963 arena_chunk_mapelms_t mapelms; 964 965 ql_new(&mapelms); 966 967 /* 968 * If chunk is the spare, temporarily re-allocate it, 1) so that its 969 * run is reinserted into runs_avail, and 2) so that it cannot be 970 * completely discarded by another thread while arena->lock is dropped 971 * by this thread. Note that the arena_run_dalloc() call will 972 * implicitly deallocate the chunk, so no explicit action is required 973 * in this function to deallocate the chunk. 974 * 975 * Note that once a chunk contains dirty pages, it cannot again contain 976 * a single run unless 1) it is a dirty run, or 2) this function purges 977 * dirty pages and causes the transition to a single clean run. Thus 978 * (chunk == arena->spare) is possible, but it is not possible for 979 * this function to be called on the spare unless it contains a dirty 980 * run. 981 */ 982 if (chunk == arena->spare) { 983 assert(arena_mapbits_dirty_get(chunk, map_bias) != 0); 984 assert(arena_mapbits_dirty_get(chunk, chunk_npages-1) != 0); 985 986 arena_chunk_alloc(arena); 987 } 988 989 if (config_stats) 990 arena->stats.purged += chunk->ndirty; 991 992 /* 993 * Operate on all dirty runs if there is no clean/dirty run 994 * fragmentation. 995 */ 996 if (chunk->nruns_adjac == 0) 997 all = true; 998 999 arena_chunk_stash_dirty(arena, chunk, all, &mapelms); 1000 npurged = arena_chunk_purge_stashed(arena, chunk, &mapelms); 1001 arena_chunk_unstash_purged(arena, chunk, &mapelms); 1002 1003 return (npurged); 1004} 1005 1006static void 1007arena_purge(arena_t *arena, bool all) 1008{ 1009 arena_chunk_t *chunk; 1010 size_t npurgatory; 1011 if (config_debug) { 1012 size_t ndirty = 0; 1013 1014 arena_chunk_dirty_iter(&arena->chunks_dirty, NULL, 1015 chunks_dirty_iter_cb, (void *)&ndirty); 1016 assert(ndirty == arena->ndirty); 1017 } 1018 assert(arena->ndirty > arena->npurgatory || all); 1019 assert((arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty - 1020 arena->npurgatory) || all); 1021 1022 if (config_stats) 1023 arena->stats.npurge++; 1024 1025 /* 1026 * Add the minimum number of pages this thread should try to purge to 1027 * arena->npurgatory. This will keep multiple threads from racing to 1028 * reduce ndirty below the threshold. 1029 */ 1030 npurgatory = arena_compute_npurgatory(arena, all); 1031 arena->npurgatory += npurgatory; 1032 1033 while (npurgatory > 0) { 1034 size_t npurgeable, npurged, nunpurged; 1035 1036 /* Get next chunk with dirty pages. */ 1037 chunk = arena_chunk_dirty_first(&arena->chunks_dirty); 1038 if (chunk == NULL) { 1039 /* 1040 * This thread was unable to purge as many pages as 1041 * originally intended, due to races with other threads 1042 * that either did some of the purging work, or re-used 1043 * dirty pages. 1044 */ 1045 arena->npurgatory -= npurgatory; 1046 return; 1047 } 1048 npurgeable = chunk->ndirty; 1049 assert(npurgeable != 0); 1050 1051 if (npurgeable > npurgatory && chunk->nruns_adjac == 0) { 1052 /* 1053 * This thread will purge all the dirty pages in chunk, 1054 * so set npurgatory to reflect this thread's intent to 1055 * purge the pages. This tends to reduce the chances 1056 * of the following scenario: 1057 * 1058 * 1) This thread sets arena->npurgatory such that 1059 * (arena->ndirty - arena->npurgatory) is at the 1060 * threshold. 1061 * 2) This thread drops arena->lock. 1062 * 3) Another thread causes one or more pages to be 1063 * dirtied, and immediately determines that it must 1064 * purge dirty pages. 1065 * 1066 * If this scenario *does* play out, that's okay, 1067 * because all of the purging work being done really 1068 * needs to happen. 1069 */ 1070 arena->npurgatory += npurgeable - npurgatory; 1071 npurgatory = npurgeable; 1072 } 1073 1074 /* 1075 * Keep track of how many pages are purgeable, versus how many 1076 * actually get purged, and adjust counters accordingly. 1077 */ 1078 arena->npurgatory -= npurgeable; 1079 npurgatory -= npurgeable; 1080 npurged = arena_chunk_purge(arena, chunk, all); 1081 nunpurged = npurgeable - npurged; 1082 arena->npurgatory += nunpurged; 1083 npurgatory += nunpurged; 1084 } 1085} 1086 1087void 1088arena_purge_all(arena_t *arena) 1089{ 1090 1091 malloc_mutex_lock(&arena->lock); 1092 arena_purge(arena, true); 1093 malloc_mutex_unlock(&arena->lock); 1094} 1095 1096static void 1097arena_run_coalesce(arena_t *arena, arena_chunk_t *chunk, size_t *p_size, 1098 size_t *p_run_ind, size_t *p_run_pages, size_t flag_dirty) 1099{ 1100 size_t size = *p_size; 1101 size_t run_ind = *p_run_ind; 1102 size_t run_pages = *p_run_pages; 1103 1104 /* Try to coalesce forward. */ 1105 if (run_ind + run_pages < chunk_npages && 1106 arena_mapbits_allocated_get(chunk, run_ind+run_pages) == 0 && 1107 arena_mapbits_dirty_get(chunk, run_ind+run_pages) == flag_dirty) { 1108 size_t nrun_size = arena_mapbits_unallocated_size_get(chunk, 1109 run_ind+run_pages); 1110 size_t nrun_pages = nrun_size >> LG_PAGE; 1111 1112 /* 1113 * Remove successor from runs_avail; the coalesced run is 1114 * inserted later. 1115 */ 1116 assert(arena_mapbits_unallocated_size_get(chunk, 1117 run_ind+run_pages+nrun_pages-1) == nrun_size); 1118 assert(arena_mapbits_dirty_get(chunk, 1119 run_ind+run_pages+nrun_pages-1) == flag_dirty); 1120 arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages, 1121 false, true); 1122 1123 size += nrun_size; 1124 run_pages += nrun_pages; 1125 1126 arena_mapbits_unallocated_size_set(chunk, run_ind, size); 1127 arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, 1128 size); 1129 } 1130 1131 /* Try to coalesce backward. */ 1132 if (run_ind > map_bias && arena_mapbits_allocated_get(chunk, 1133 run_ind-1) == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) == 1134 flag_dirty) { 1135 size_t prun_size = arena_mapbits_unallocated_size_get(chunk, 1136 run_ind-1); 1137 size_t prun_pages = prun_size >> LG_PAGE; 1138 1139 run_ind -= prun_pages; 1140 1141 /* 1142 * Remove predecessor from runs_avail; the coalesced run is 1143 * inserted later. 1144 */ 1145 assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == 1146 prun_size); 1147 assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty); 1148 arena_avail_remove(arena, chunk, run_ind, prun_pages, true, 1149 false); 1150 1151 size += prun_size; 1152 run_pages += prun_pages; 1153 1154 arena_mapbits_unallocated_size_set(chunk, run_ind, size); 1155 arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, 1156 size); 1157 } 1158 1159 *p_size = size; 1160 *p_run_ind = run_ind; 1161 *p_run_pages = run_pages; 1162} 1163 1164static void 1165arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) 1166{ 1167 arena_chunk_t *chunk; 1168 size_t size, run_ind, run_pages, flag_dirty; 1169 1170 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1171 run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 1172 assert(run_ind >= map_bias); 1173 assert(run_ind < chunk_npages); 1174 if (arena_mapbits_large_get(chunk, run_ind) != 0) { 1175 size = arena_mapbits_large_size_get(chunk, run_ind); 1176 assert(size == PAGE || 1177 arena_mapbits_large_size_get(chunk, 1178 run_ind+(size>>LG_PAGE)-1) == 0); 1179 } else { 1180 size_t binind = arena_bin_index(arena, run->bin); 1181 arena_bin_info_t *bin_info = &arena_bin_info[binind]; 1182 size = bin_info->run_size; 1183 } 1184 run_pages = (size >> LG_PAGE); 1185 arena_cactive_update(arena, 0, run_pages); 1186 arena->nactive -= run_pages; 1187 1188 /* 1189 * The run is dirty if the caller claims to have dirtied it, as well as 1190 * if it was already dirty before being allocated and the caller 1191 * doesn't claim to have cleaned it. 1192 */ 1193 assert(arena_mapbits_dirty_get(chunk, run_ind) == 1194 arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); 1195 if (cleaned == false && arena_mapbits_dirty_get(chunk, run_ind) != 0) 1196 dirty = true; 1197 flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; 1198 1199 /* Mark pages as unallocated in the chunk map. */ 1200 if (dirty) { 1201 arena_mapbits_unallocated_set(chunk, run_ind, size, 1202 CHUNK_MAP_DIRTY); 1203 arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, 1204 CHUNK_MAP_DIRTY); 1205 } else { 1206 arena_mapbits_unallocated_set(chunk, run_ind, size, 1207 arena_mapbits_unzeroed_get(chunk, run_ind)); 1208 arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, 1209 arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1)); 1210 } 1211 1212 arena_run_coalesce(arena, chunk, &size, &run_ind, &run_pages, 1213 flag_dirty); 1214 1215 /* Insert into runs_avail, now that coalescing is complete. */ 1216 assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == 1217 arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1)); 1218 assert(arena_mapbits_dirty_get(chunk, run_ind) == 1219 arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); 1220 arena_avail_insert(arena, chunk, run_ind, run_pages, true, true); 1221 1222 /* Deallocate chunk if it is now completely unused. */ 1223 if (size == arena_maxclass) { 1224 assert(run_ind == map_bias); 1225 assert(run_pages == (arena_maxclass >> LG_PAGE)); 1226 arena_chunk_dealloc(arena, chunk); 1227 } 1228 1229 /* 1230 * It is okay to do dirty page processing here even if the chunk was 1231 * deallocated above, since in that case it is the spare. Waiting 1232 * until after possible chunk deallocation to do dirty processing 1233 * allows for an old spare to be fully deallocated, thus decreasing the 1234 * chances of spuriously crossing the dirty page purging threshold. 1235 */ 1236 if (dirty) 1237 arena_maybe_purge(arena); 1238} 1239 1240static void 1241arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1242 size_t oldsize, size_t newsize) 1243{ 1244 size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1245 size_t head_npages = (oldsize - newsize) >> LG_PAGE; 1246 size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); 1247 1248 assert(oldsize > newsize); 1249 1250 /* 1251 * Update the chunk map so that arena_run_dalloc() can treat the 1252 * leading run as separately allocated. Set the last element of each 1253 * run first, in case of single-page runs. 1254 */ 1255 assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); 1256 arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty); 1257 arena_mapbits_large_set(chunk, pageind, oldsize-newsize, flag_dirty); 1258 1259 if (config_debug) { 1260 UNUSED size_t tail_npages = newsize >> LG_PAGE; 1261 assert(arena_mapbits_large_size_get(chunk, 1262 pageind+head_npages+tail_npages-1) == 0); 1263 assert(arena_mapbits_dirty_get(chunk, 1264 pageind+head_npages+tail_npages-1) == flag_dirty); 1265 } 1266 arena_mapbits_large_set(chunk, pageind+head_npages, newsize, 1267 flag_dirty); 1268 1269 arena_run_dalloc(arena, run, false, false); 1270} 1271 1272static void 1273arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1274 size_t oldsize, size_t newsize, bool dirty) 1275{ 1276 size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1277 size_t head_npages = newsize >> LG_PAGE; 1278 size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); 1279 1280 assert(oldsize > newsize); 1281 1282 /* 1283 * Update the chunk map so that arena_run_dalloc() can treat the 1284 * trailing run as separately allocated. Set the last element of each 1285 * run first, in case of single-page runs. 1286 */ 1287 assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); 1288 arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty); 1289 arena_mapbits_large_set(chunk, pageind, newsize, flag_dirty); 1290 1291 if (config_debug) { 1292 UNUSED size_t tail_npages = (oldsize - newsize) >> LG_PAGE; 1293 assert(arena_mapbits_large_size_get(chunk, 1294 pageind+head_npages+tail_npages-1) == 0); 1295 assert(arena_mapbits_dirty_get(chunk, 1296 pageind+head_npages+tail_npages-1) == flag_dirty); 1297 } 1298 arena_mapbits_large_set(chunk, pageind+head_npages, oldsize-newsize, 1299 flag_dirty); 1300 1301 arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize), 1302 dirty, false); 1303} 1304 1305static arena_run_t * 1306arena_bin_runs_first(arena_bin_t *bin) 1307{ 1308 arena_chunk_map_t *mapelm = arena_run_tree_first(&bin->runs); 1309 if (mapelm != NULL) { 1310 arena_chunk_t *chunk; 1311 size_t pageind; 1312 arena_run_t *run; 1313 1314 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); 1315 pageind = arena_mapelm_to_pageind(mapelm); 1316 run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 1317 arena_mapbits_small_runind_get(chunk, pageind)) << 1318 LG_PAGE)); 1319 return (run); 1320 } 1321 1322 return (NULL); 1323} 1324 1325static void 1326arena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run) 1327{ 1328 arena_chunk_t *chunk = CHUNK_ADDR2BASE(run); 1329 size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1330 arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); 1331 1332 assert(arena_run_tree_search(&bin->runs, mapelm) == NULL); 1333 1334 arena_run_tree_insert(&bin->runs, mapelm); 1335} 1336 1337static void 1338arena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run) 1339{ 1340 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1341 size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1342 arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); 1343 1344 assert(arena_run_tree_search(&bin->runs, mapelm) != NULL); 1345 1346 arena_run_tree_remove(&bin->runs, mapelm); 1347} 1348 1349static arena_run_t * 1350arena_bin_nonfull_run_tryget(arena_bin_t *bin) 1351{ 1352 arena_run_t *run = arena_bin_runs_first(bin); 1353 if (run != NULL) { 1354 arena_bin_runs_remove(bin, run); 1355 if (config_stats) 1356 bin->stats.reruns++; 1357 } 1358 return (run); 1359} 1360 1361static arena_run_t * 1362arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) 1363{ 1364 arena_run_t *run; 1365 size_t binind; 1366 arena_bin_info_t *bin_info; 1367 1368 /* Look for a usable run. */ 1369 run = arena_bin_nonfull_run_tryget(bin); 1370 if (run != NULL) 1371 return (run); 1372 /* No existing runs have any space available. */ 1373 1374 binind = arena_bin_index(arena, bin); 1375 bin_info = &arena_bin_info[binind]; 1376 1377 /* Allocate a new run. */ 1378 malloc_mutex_unlock(&bin->lock); 1379 /******************************/ 1380 malloc_mutex_lock(&arena->lock); 1381 run = arena_run_alloc_small(arena, bin_info->run_size, binind); 1382 if (run != NULL) { 1383 bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 1384 (uintptr_t)bin_info->bitmap_offset); 1385 1386 /* Initialize run internals. */ 1387 run->bin = bin; 1388 run->nextind = 0; 1389 run->nfree = bin_info->nregs; 1390 bitmap_init(bitmap, &bin_info->bitmap_info); 1391 } 1392 malloc_mutex_unlock(&arena->lock); 1393 /********************************/ 1394 malloc_mutex_lock(&bin->lock); 1395 if (run != NULL) { 1396 if (config_stats) { 1397 bin->stats.nruns++; 1398 bin->stats.curruns++; 1399 } 1400 return (run); 1401 } 1402 1403 /* 1404 * arena_run_alloc_small() failed, but another thread may have made 1405 * sufficient memory available while this one dropped bin->lock above, 1406 * so search one more time. 1407 */ 1408 run = arena_bin_nonfull_run_tryget(bin); 1409 if (run != NULL) 1410 return (run); 1411 1412 return (NULL); 1413} 1414 1415/* Re-fill bin->runcur, then call arena_run_reg_alloc(). */ 1416static void * 1417arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) 1418{ 1419 void *ret; 1420 size_t binind; 1421 arena_bin_info_t *bin_info; 1422 arena_run_t *run; 1423 1424 binind = arena_bin_index(arena, bin); 1425 bin_info = &arena_bin_info[binind]; 1426 bin->runcur = NULL; 1427 run = arena_bin_nonfull_run_get(arena, bin); 1428 if (bin->runcur != NULL && bin->runcur->nfree > 0) { 1429 /* 1430 * Another thread updated runcur while this one ran without the 1431 * bin lock in arena_bin_nonfull_run_get(). 1432 */ 1433 assert(bin->runcur->nfree > 0); 1434 ret = arena_run_reg_alloc(bin->runcur, bin_info); 1435 if (run != NULL) { 1436 arena_chunk_t *chunk; 1437 1438 /* 1439 * arena_run_alloc_small() may have allocated run, or 1440 * it may have pulled run from the bin's run tree. 1441 * Therefore it is unsafe to make any assumptions about 1442 * how run has previously been used, and 1443 * arena_bin_lower_run() must be called, as if a region 1444 * were just deallocated from the run. 1445 */ 1446 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1447 if (run->nfree == bin_info->nregs) 1448 arena_dalloc_bin_run(arena, chunk, run, bin); 1449 else 1450 arena_bin_lower_run(arena, chunk, run, bin); 1451 } 1452 return (ret); 1453 } 1454 1455 if (run == NULL) 1456 return (NULL); 1457 1458 bin->runcur = run; 1459 1460 assert(bin->runcur->nfree > 0); 1461 1462 return (arena_run_reg_alloc(bin->runcur, bin_info)); 1463} 1464 1465void 1466arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind, 1467 uint64_t prof_accumbytes) 1468{ 1469 unsigned i, nfill; 1470 arena_bin_t *bin; 1471 arena_run_t *run; 1472 void *ptr; 1473 1474 assert(tbin->ncached == 0); 1475 1476 if (config_prof && arena_prof_accum(arena, prof_accumbytes)) 1477 prof_idump(); 1478 bin = &arena->bins[binind]; 1479 malloc_mutex_lock(&bin->lock); 1480 for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> 1481 tbin->lg_fill_div); i < nfill; i++) { 1482 if ((run = bin->runcur) != NULL && run->nfree > 0) 1483 ptr = arena_run_reg_alloc(run, &arena_bin_info[binind]); 1484 else 1485 ptr = arena_bin_malloc_hard(arena, bin); 1486 if (ptr == NULL) 1487 break; 1488 if (config_fill && opt_junk) { 1489 arena_alloc_junk_small(ptr, &arena_bin_info[binind], 1490 true); 1491 } 1492 /* Insert such that low regions get used first. */ 1493 tbin->avail[nfill - 1 - i] = ptr; 1494 } 1495 if (config_stats) { 1496 bin->stats.allocated += i * arena_bin_info[binind].reg_size; 1497 bin->stats.nmalloc += i; 1498 bin->stats.nrequests += tbin->tstats.nrequests; 1499 bin->stats.nfills++; 1500 tbin->tstats.nrequests = 0; 1501 } 1502 malloc_mutex_unlock(&bin->lock); 1503 tbin->ncached = i; 1504} 1505 1506void 1507arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, bool zero) 1508{ 1509 1510 if (zero) { 1511 size_t redzone_size = bin_info->redzone_size; 1512 memset((void *)((uintptr_t)ptr - redzone_size), 0xa5, 1513 redzone_size); 1514 memset((void *)((uintptr_t)ptr + bin_info->reg_size), 0xa5, 1515 redzone_size); 1516 } else { 1517 memset((void *)((uintptr_t)ptr - bin_info->redzone_size), 0xa5, 1518 bin_info->reg_interval); 1519 } 1520} 1521 1522#ifdef JEMALLOC_JET 1523#undef arena_redzone_corruption 1524#define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption_impl) 1525#endif 1526static void 1527arena_redzone_corruption(void *ptr, size_t usize, bool after, 1528 size_t offset, uint8_t byte) 1529{ 1530 1531 malloc_printf("<jemalloc>: Corrupt redzone %zu byte%s %s %p " 1532 "(size %zu), byte=%#x\n", offset, (offset == 1) ? "" : "s", 1533 after ? "after" : "before", ptr, usize, byte); 1534} 1535#ifdef JEMALLOC_JET 1536#undef arena_redzone_corruption 1537#define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption) 1538arena_redzone_corruption_t *arena_redzone_corruption = 1539 JEMALLOC_N(arena_redzone_corruption_impl); 1540#endif 1541 1542static void 1543arena_redzones_validate(void *ptr, arena_bin_info_t *bin_info, bool reset) 1544{ 1545 size_t size = bin_info->reg_size; 1546 size_t redzone_size = bin_info->redzone_size; 1547 size_t i; 1548 bool error = false; 1549 1550 for (i = 1; i <= redzone_size; i++) { 1551 uint8_t *byte = (uint8_t *)((uintptr_t)ptr - i); 1552 if (*byte != 0xa5) { 1553 error = true; 1554 arena_redzone_corruption(ptr, size, false, i, *byte); 1555 if (reset) 1556 *byte = 0xa5; 1557 } 1558 } 1559 for (i = 0; i < redzone_size; i++) { 1560 uint8_t *byte = (uint8_t *)((uintptr_t)ptr + size + i); 1561 if (*byte != 0xa5) { 1562 error = true; 1563 arena_redzone_corruption(ptr, size, true, i, *byte); 1564 if (reset) 1565 *byte = 0xa5; 1566 } 1567 } 1568 if (opt_abort && error) 1569 abort(); 1570} 1571 1572#ifdef JEMALLOC_JET 1573#undef arena_dalloc_junk_small 1574#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small_impl) 1575#endif 1576void 1577arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info) 1578{ 1579 size_t redzone_size = bin_info->redzone_size; 1580 1581 arena_redzones_validate(ptr, bin_info, false); 1582 memset((void *)((uintptr_t)ptr - redzone_size), 0x5a, 1583 bin_info->reg_interval); 1584} 1585#ifdef JEMALLOC_JET 1586#undef arena_dalloc_junk_small 1587#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small) 1588arena_dalloc_junk_small_t *arena_dalloc_junk_small = 1589 JEMALLOC_N(arena_dalloc_junk_small_impl); 1590#endif 1591 1592void 1593arena_quarantine_junk_small(void *ptr, size_t usize) 1594{ 1595 size_t binind; 1596 arena_bin_info_t *bin_info; 1597 cassert(config_fill); 1598 assert(opt_junk); 1599 assert(opt_quarantine); 1600 assert(usize <= SMALL_MAXCLASS); 1601 1602 binind = SMALL_SIZE2BIN(usize); 1603 bin_info = &arena_bin_info[binind]; 1604 arena_redzones_validate(ptr, bin_info, true); 1605} 1606 1607void * 1608arena_malloc_small(arena_t *arena, size_t size, bool zero) 1609{ 1610 void *ret; 1611 arena_bin_t *bin; 1612 arena_run_t *run; 1613 size_t binind; 1614 1615 binind = SMALL_SIZE2BIN(size); 1616 assert(binind < NBINS); 1617 bin = &arena->bins[binind]; 1618 size = arena_bin_info[binind].reg_size; 1619 1620 malloc_mutex_lock(&bin->lock); 1621 if ((run = bin->runcur) != NULL && run->nfree > 0) 1622 ret = arena_run_reg_alloc(run, &arena_bin_info[binind]); 1623 else 1624 ret = arena_bin_malloc_hard(arena, bin); 1625 1626 if (ret == NULL) { 1627 malloc_mutex_unlock(&bin->lock); 1628 return (NULL); 1629 } 1630 1631 if (config_stats) { 1632 bin->stats.allocated += size; 1633 bin->stats.nmalloc++; 1634 bin->stats.nrequests++; 1635 } 1636 malloc_mutex_unlock(&bin->lock); 1637 if (config_prof && isthreaded == false && arena_prof_accum(arena, size)) 1638 prof_idump(); 1639 1640 if (zero == false) { 1641 if (config_fill) { 1642 if (opt_junk) { 1643 arena_alloc_junk_small(ret, 1644 &arena_bin_info[binind], false); 1645 } else if (opt_zero) 1646 memset(ret, 0, size); 1647 } 1648 VALGRIND_MAKE_MEM_UNDEFINED(ret, size); 1649 } else { 1650 if (config_fill && opt_junk) { 1651 arena_alloc_junk_small(ret, &arena_bin_info[binind], 1652 true); 1653 } 1654 VALGRIND_MAKE_MEM_UNDEFINED(ret, size); 1655 memset(ret, 0, size); 1656 } 1657 1658 return (ret); 1659} 1660 1661void * 1662arena_malloc_large(arena_t *arena, size_t size, bool zero) 1663{ 1664 void *ret; 1665 UNUSED bool idump; 1666 1667 /* Large allocation. */ 1668 size = PAGE_CEILING(size); 1669 malloc_mutex_lock(&arena->lock); 1670 ret = (void *)arena_run_alloc_large(arena, size, zero); 1671 if (ret == NULL) { 1672 malloc_mutex_unlock(&arena->lock); 1673 return (NULL); 1674 } 1675 if (config_stats) { 1676 arena->stats.nmalloc_large++; 1677 arena->stats.nrequests_large++; 1678 arena->stats.allocated_large += size; 1679 arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 1680 arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 1681 arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 1682 } 1683 if (config_prof) 1684 idump = arena_prof_accum_locked(arena, size); 1685 malloc_mutex_unlock(&arena->lock); 1686 if (config_prof && idump) 1687 prof_idump(); 1688 1689 if (zero == false) { 1690 if (config_fill) { 1691 if (opt_junk) 1692 memset(ret, 0xa5, size); 1693 else if (opt_zero) 1694 memset(ret, 0, size); 1695 } 1696 } 1697 1698 return (ret); 1699} 1700 1701/* Only handles large allocations that require more than page alignment. */ 1702void * 1703arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero) 1704{ 1705 void *ret; 1706 size_t alloc_size, leadsize, trailsize; 1707 arena_run_t *run; 1708 arena_chunk_t *chunk; 1709 1710 assert((size & PAGE_MASK) == 0); 1711 1712 alignment = PAGE_CEILING(alignment); 1713 alloc_size = size + alignment - PAGE; 1714 1715 malloc_mutex_lock(&arena->lock); 1716 run = arena_run_alloc_large(arena, alloc_size, false); 1717 if (run == NULL) { 1718 malloc_mutex_unlock(&arena->lock); 1719 return (NULL); 1720 } 1721 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1722 1723 leadsize = ALIGNMENT_CEILING((uintptr_t)run, alignment) - 1724 (uintptr_t)run; 1725 assert(alloc_size >= leadsize + size); 1726 trailsize = alloc_size - leadsize - size; 1727 ret = (void *)((uintptr_t)run + leadsize); 1728 if (leadsize != 0) { 1729 arena_run_trim_head(arena, chunk, run, alloc_size, alloc_size - 1730 leadsize); 1731 } 1732 if (trailsize != 0) { 1733 arena_run_trim_tail(arena, chunk, ret, size + trailsize, size, 1734 false); 1735 } 1736 arena_run_init_large(arena, (arena_run_t *)ret, size, zero); 1737 1738 if (config_stats) { 1739 arena->stats.nmalloc_large++; 1740 arena->stats.nrequests_large++; 1741 arena->stats.allocated_large += size; 1742 arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 1743 arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 1744 arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 1745 } 1746 malloc_mutex_unlock(&arena->lock); 1747 1748 if (config_fill && zero == false) { 1749 if (opt_junk) 1750 memset(ret, 0xa5, size); 1751 else if (opt_zero) 1752 memset(ret, 0, size); 1753 } 1754 return (ret); 1755} 1756 1757void 1758arena_prof_promoted(const void *ptr, size_t size) 1759{ 1760 arena_chunk_t *chunk; 1761 size_t pageind, binind; 1762 1763 cassert(config_prof); 1764 assert(ptr != NULL); 1765 assert(CHUNK_ADDR2BASE(ptr) != ptr); 1766 assert(isalloc(ptr, false) == PAGE); 1767 assert(isalloc(ptr, true) == PAGE); 1768 assert(size <= SMALL_MAXCLASS); 1769 1770 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1771 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1772 binind = SMALL_SIZE2BIN(size); 1773 assert(binind < NBINS); 1774 arena_mapbits_large_binind_set(chunk, pageind, binind); 1775 1776 assert(isalloc(ptr, false) == PAGE); 1777 assert(isalloc(ptr, true) == size); 1778} 1779 1780static void 1781arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, 1782 arena_bin_t *bin) 1783{ 1784 1785 /* Dissociate run from bin. */ 1786 if (run == bin->runcur) 1787 bin->runcur = NULL; 1788 else { 1789 size_t binind = arena_bin_index(chunk->arena, bin); 1790 arena_bin_info_t *bin_info = &arena_bin_info[binind]; 1791 1792 if (bin_info->nregs != 1) { 1793 /* 1794 * This block's conditional is necessary because if the 1795 * run only contains one region, then it never gets 1796 * inserted into the non-full runs tree. 1797 */ 1798 arena_bin_runs_remove(bin, run); 1799 } 1800 } 1801} 1802 1803static void 1804arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1805 arena_bin_t *bin) 1806{ 1807 size_t binind; 1808 arena_bin_info_t *bin_info; 1809 size_t npages, run_ind, past; 1810 1811 assert(run != bin->runcur); 1812 assert(arena_run_tree_search(&bin->runs, 1813 arena_mapp_get(chunk, ((uintptr_t)run-(uintptr_t)chunk)>>LG_PAGE)) 1814 == NULL); 1815 1816 binind = arena_bin_index(chunk->arena, run->bin); 1817 bin_info = &arena_bin_info[binind]; 1818 1819 malloc_mutex_unlock(&bin->lock); 1820 /******************************/ 1821 npages = bin_info->run_size >> LG_PAGE; 1822 run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 1823 past = (size_t)(PAGE_CEILING((uintptr_t)run + 1824 (uintptr_t)bin_info->reg0_offset + (uintptr_t)(run->nextind * 1825 bin_info->reg_interval - bin_info->redzone_size) - 1826 (uintptr_t)chunk) >> LG_PAGE); 1827 malloc_mutex_lock(&arena->lock); 1828 1829 /* 1830 * If the run was originally clean, and some pages were never touched, 1831 * trim the clean pages before deallocating the dirty portion of the 1832 * run. 1833 */ 1834 assert(arena_mapbits_dirty_get(chunk, run_ind) == 1835 arena_mapbits_dirty_get(chunk, run_ind+npages-1)); 1836 if (arena_mapbits_dirty_get(chunk, run_ind) == 0 && past - run_ind < 1837 npages) { 1838 /* Trim clean pages. Convert to large run beforehand. */ 1839 assert(npages > 0); 1840 arena_mapbits_large_set(chunk, run_ind, bin_info->run_size, 0); 1841 arena_mapbits_large_set(chunk, run_ind+npages-1, 0, 0); 1842 arena_run_trim_tail(arena, chunk, run, (npages << LG_PAGE), 1843 ((past - run_ind) << LG_PAGE), false); 1844 /* npages = past - run_ind; */ 1845 } 1846 arena_run_dalloc(arena, run, true, false); 1847 malloc_mutex_unlock(&arena->lock); 1848 /****************************/ 1849 malloc_mutex_lock(&bin->lock); 1850 if (config_stats) 1851 bin->stats.curruns--; 1852} 1853 1854static void 1855arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1856 arena_bin_t *bin) 1857{ 1858 1859 /* 1860 * Make sure that if bin->runcur is non-NULL, it refers to the lowest 1861 * non-full run. It is okay to NULL runcur out rather than proactively 1862 * keeping it pointing at the lowest non-full run. 1863 */ 1864 if ((uintptr_t)run < (uintptr_t)bin->runcur) { 1865 /* Switch runcur. */ 1866 if (bin->runcur->nfree > 0) 1867 arena_bin_runs_insert(bin, bin->runcur); 1868 bin->runcur = run; 1869 if (config_stats) 1870 bin->stats.reruns++; 1871 } else 1872 arena_bin_runs_insert(bin, run); 1873} 1874 1875void 1876arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1877 arena_chunk_map_t *mapelm) 1878{ 1879 size_t pageind; 1880 arena_run_t *run; 1881 arena_bin_t *bin; 1882 arena_bin_info_t *bin_info; 1883 size_t size, binind; 1884 1885 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1886 run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 1887 arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); 1888 bin = run->bin; 1889 binind = arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, pageind)); 1890 bin_info = &arena_bin_info[binind]; 1891 if (config_fill || config_stats) 1892 size = bin_info->reg_size; 1893 1894 if (config_fill && opt_junk) 1895 arena_dalloc_junk_small(ptr, bin_info); 1896 1897 arena_run_reg_dalloc(run, ptr); 1898 if (run->nfree == bin_info->nregs) { 1899 arena_dissociate_bin_run(chunk, run, bin); 1900 arena_dalloc_bin_run(arena, chunk, run, bin); 1901 } else if (run->nfree == 1 && run != bin->runcur) 1902 arena_bin_lower_run(arena, chunk, run, bin); 1903 1904 if (config_stats) { 1905 bin->stats.allocated -= size; 1906 bin->stats.ndalloc++; 1907 } 1908} 1909 1910void 1911arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1912 size_t pageind, arena_chunk_map_t *mapelm) 1913{ 1914 arena_run_t *run; 1915 arena_bin_t *bin; 1916 1917 run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 1918 arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); 1919 bin = run->bin; 1920 malloc_mutex_lock(&bin->lock); 1921 arena_dalloc_bin_locked(arena, chunk, ptr, mapelm); 1922 malloc_mutex_unlock(&bin->lock); 1923} 1924 1925void 1926arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1927 size_t pageind) 1928{ 1929 arena_chunk_map_t *mapelm; 1930 1931 if (config_debug) { 1932 /* arena_ptr_small_binind_get() does extra sanity checking. */ 1933 assert(arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, 1934 pageind)) != BININD_INVALID); 1935 } 1936 mapelm = arena_mapp_get(chunk, pageind); 1937 arena_dalloc_bin(arena, chunk, ptr, pageind, mapelm); 1938} 1939 1940#ifdef JEMALLOC_JET 1941#undef arena_dalloc_junk_large 1942#define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large_impl) 1943#endif 1944static void 1945arena_dalloc_junk_large(void *ptr, size_t usize) 1946{ 1947 1948 if (config_fill && opt_junk) 1949 memset(ptr, 0x5a, usize); 1950} 1951#ifdef JEMALLOC_JET 1952#undef arena_dalloc_junk_large 1953#define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large) 1954arena_dalloc_junk_large_t *arena_dalloc_junk_large = 1955 JEMALLOC_N(arena_dalloc_junk_large_impl); 1956#endif 1957 1958void 1959arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr) 1960{ 1961 1962 if (config_fill || config_stats) { 1963 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1964 size_t usize = arena_mapbits_large_size_get(chunk, pageind); 1965 1966 arena_dalloc_junk_large(ptr, usize); 1967 if (config_stats) { 1968 arena->stats.ndalloc_large++; 1969 arena->stats.allocated_large -= usize; 1970 arena->stats.lstats[(usize >> LG_PAGE) - 1].ndalloc++; 1971 arena->stats.lstats[(usize >> LG_PAGE) - 1].curruns--; 1972 } 1973 } 1974 1975 arena_run_dalloc(arena, (arena_run_t *)ptr, true, false); 1976} 1977 1978void 1979arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr) 1980{ 1981 1982 malloc_mutex_lock(&arena->lock); 1983 arena_dalloc_large_locked(arena, chunk, ptr); 1984 malloc_mutex_unlock(&arena->lock); 1985} 1986 1987static void 1988arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1989 size_t oldsize, size_t size) 1990{ 1991 1992 assert(size < oldsize); 1993 1994 /* 1995 * Shrink the run, and make trailing pages available for other 1996 * allocations. 1997 */ 1998 malloc_mutex_lock(&arena->lock); 1999 arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size, 2000 true); 2001 if (config_stats) { 2002 arena->stats.ndalloc_large++; 2003 arena->stats.allocated_large -= oldsize; 2004 arena->stats.lstats[(oldsize >> LG_PAGE) - 1].ndalloc++; 2005 arena->stats.lstats[(oldsize >> LG_PAGE) - 1].curruns--; 2006 2007 arena->stats.nmalloc_large++; 2008 arena->stats.nrequests_large++; 2009 arena->stats.allocated_large += size; 2010 arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 2011 arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 2012 arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 2013 } 2014 malloc_mutex_unlock(&arena->lock); 2015} 2016 2017static bool 2018arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, 2019 size_t oldsize, size_t size, size_t extra, bool zero) 2020{ 2021 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 2022 size_t npages = oldsize >> LG_PAGE; 2023 size_t followsize; 2024 2025 assert(oldsize == arena_mapbits_large_size_get(chunk, pageind)); 2026 2027 /* Try to extend the run. */ 2028 assert(size + extra > oldsize); 2029 malloc_mutex_lock(&arena->lock); 2030 if (pageind + npages < chunk_npages && 2031 arena_mapbits_allocated_get(chunk, pageind+npages) == 0 && 2032 (followsize = arena_mapbits_unallocated_size_get(chunk, 2033 pageind+npages)) >= size - oldsize) { 2034 /* 2035 * The next run is available and sufficiently large. Split the 2036 * following run, then merge the first part with the existing 2037 * allocation. 2038 */ 2039 size_t flag_dirty; 2040 size_t splitsize = (oldsize + followsize <= size + extra) 2041 ? followsize : size + extra - oldsize; 2042 arena_run_split_large(arena, (arena_run_t *)((uintptr_t)chunk + 2043 ((pageind+npages) << LG_PAGE)), splitsize, zero); 2044 2045 size = oldsize + splitsize; 2046 npages = size >> LG_PAGE; 2047 2048 /* 2049 * Mark the extended run as dirty if either portion of the run 2050 * was dirty before allocation. This is rather pedantic, 2051 * because there's not actually any sequence of events that 2052 * could cause the resulting run to be passed to 2053 * arena_run_dalloc() with the dirty argument set to false 2054 * (which is when dirty flag consistency would really matter). 2055 */ 2056 flag_dirty = arena_mapbits_dirty_get(chunk, pageind) | 2057 arena_mapbits_dirty_get(chunk, pageind+npages-1); 2058 arena_mapbits_large_set(chunk, pageind, size, flag_dirty); 2059 arena_mapbits_large_set(chunk, pageind+npages-1, 0, flag_dirty); 2060 2061 if (config_stats) { 2062 arena->stats.ndalloc_large++; 2063 arena->stats.allocated_large -= oldsize; 2064 arena->stats.lstats[(oldsize >> LG_PAGE) - 1].ndalloc++; 2065 arena->stats.lstats[(oldsize >> LG_PAGE) - 1].curruns--; 2066 2067 arena->stats.nmalloc_large++; 2068 arena->stats.nrequests_large++; 2069 arena->stats.allocated_large += size; 2070 arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 2071 arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 2072 arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 2073 } 2074 malloc_mutex_unlock(&arena->lock); 2075 return (false); 2076 } 2077 malloc_mutex_unlock(&arena->lock); 2078 2079 return (true); 2080} 2081 2082#ifdef JEMALLOC_JET 2083#undef arena_ralloc_junk_large 2084#define arena_ralloc_junk_large JEMALLOC_N(arena_ralloc_junk_large_impl) 2085#endif 2086static void 2087arena_ralloc_junk_large(void *ptr, size_t old_usize, size_t usize) 2088{ 2089 2090 if (config_fill && opt_junk) { 2091 memset((void *)((uintptr_t)ptr + usize), 0x5a, 2092 old_usize - usize); 2093 } 2094} 2095#ifdef JEMALLOC_JET 2096#undef arena_ralloc_junk_large 2097#define arena_ralloc_junk_large JEMALLOC_N(arena_ralloc_junk_large) 2098arena_ralloc_junk_large_t *arena_ralloc_junk_large = 2099 JEMALLOC_N(arena_ralloc_junk_large_impl); 2100#endif 2101 2102/* 2103 * Try to resize a large allocation, in order to avoid copying. This will 2104 * always fail if growing an object, and the following run is already in use. 2105 */ 2106static bool 2107arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, 2108 bool zero) 2109{ 2110 size_t psize; 2111 2112 psize = PAGE_CEILING(size + extra); 2113 if (psize == oldsize) { 2114 /* Same size class. */ 2115 return (false); 2116 } else { 2117 arena_chunk_t *chunk; 2118 arena_t *arena; 2119 2120 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 2121 arena = chunk->arena; 2122 2123 if (psize < oldsize) { 2124 /* Fill before shrinking in order avoid a race. */ 2125 arena_ralloc_junk_large(ptr, oldsize, psize); 2126 arena_ralloc_large_shrink(arena, chunk, ptr, oldsize, 2127 psize); 2128 return (false); 2129 } else { 2130 bool ret = arena_ralloc_large_grow(arena, chunk, ptr, 2131 oldsize, PAGE_CEILING(size), 2132 psize - PAGE_CEILING(size), zero); 2133 if (config_fill && ret == false && zero == false) { 2134 if (opt_junk) { 2135 memset((void *)((uintptr_t)ptr + 2136 oldsize), 0xa5, isalloc(ptr, 2137 config_prof) - oldsize); 2138 } else if (opt_zero) { 2139 memset((void *)((uintptr_t)ptr + 2140 oldsize), 0, isalloc(ptr, 2141 config_prof) - oldsize); 2142 } 2143 } 2144 return (ret); 2145 } 2146 } 2147} 2148 2149bool 2150arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, 2151 bool zero) 2152{ 2153 2154 /* 2155 * Avoid moving the allocation if the size class can be left the same. 2156 */ 2157 if (oldsize <= arena_maxclass) { 2158 if (oldsize <= SMALL_MAXCLASS) { 2159 assert(arena_bin_info[SMALL_SIZE2BIN(oldsize)].reg_size 2160 == oldsize); 2161 if ((size + extra <= SMALL_MAXCLASS && 2162 SMALL_SIZE2BIN(size + extra) == 2163 SMALL_SIZE2BIN(oldsize)) || (size <= oldsize && 2164 size + extra >= oldsize)) 2165 return (false); 2166 } else { 2167 assert(size <= arena_maxclass); 2168 if (size + extra > SMALL_MAXCLASS) { 2169 if (arena_ralloc_large(ptr, oldsize, size, 2170 extra, zero) == false) 2171 return (false); 2172 } 2173 } 2174 } 2175 2176 /* Reallocation would require a move. */ 2177 return (true); 2178} 2179 2180void * 2181arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, 2182 size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, 2183 bool try_tcache_dalloc) 2184{ 2185 void *ret; 2186 size_t copysize; 2187 2188 /* Try to avoid moving the allocation. */ 2189 if (arena_ralloc_no_move(ptr, oldsize, size, extra, zero) == false) 2190 return (ptr); 2191 2192 /* 2193 * size and oldsize are different enough that we need to move the 2194 * object. In that case, fall back to allocating new space and 2195 * copying. 2196 */ 2197 if (alignment != 0) { 2198 size_t usize = sa2u(size + extra, alignment); 2199 if (usize == 0) 2200 return (NULL); 2201 ret = ipalloct(usize, alignment, zero, try_tcache_alloc, arena); 2202 } else 2203 ret = arena_malloc(arena, size + extra, zero, try_tcache_alloc); 2204 2205 if (ret == NULL) { 2206 if (extra == 0) 2207 return (NULL); 2208 /* Try again, this time without extra. */ 2209 if (alignment != 0) { 2210 size_t usize = sa2u(size, alignment); 2211 if (usize == 0) 2212 return (NULL); 2213 ret = ipalloct(usize, alignment, zero, try_tcache_alloc, 2214 arena); 2215 } else 2216 ret = arena_malloc(arena, size, zero, try_tcache_alloc); 2217 2218 if (ret == NULL) 2219 return (NULL); 2220 } 2221 2222 /* Junk/zero-filling were already done by ipalloc()/arena_malloc(). */ 2223 2224 /* 2225 * Copy at most size bytes (not size+extra), since the caller has no 2226 * expectation that the extra bytes will be reliably preserved. 2227 */ 2228 copysize = (size < oldsize) ? size : oldsize; 2229 VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize); 2230 memcpy(ret, ptr, copysize); 2231 iqalloct(ptr, try_tcache_dalloc); 2232 return (ret); 2233} 2234 2235dss_prec_t 2236arena_dss_prec_get(arena_t *arena) 2237{ 2238 dss_prec_t ret; 2239 2240 malloc_mutex_lock(&arena->lock); 2241 ret = arena->dss_prec; 2242 malloc_mutex_unlock(&arena->lock); 2243 return (ret); 2244} 2245 2246void 2247arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) 2248{ 2249 2250 malloc_mutex_lock(&arena->lock); 2251 arena->dss_prec = dss_prec; 2252 malloc_mutex_unlock(&arena->lock); 2253} 2254 2255void 2256arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, 2257 size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, 2258 malloc_large_stats_t *lstats) 2259{ 2260 unsigned i; 2261 2262 malloc_mutex_lock(&arena->lock); 2263 *dss = dss_prec_names[arena->dss_prec]; 2264 *nactive += arena->nactive; 2265 *ndirty += arena->ndirty; 2266 2267 astats->mapped += arena->stats.mapped; 2268 astats->npurge += arena->stats.npurge; 2269 astats->nmadvise += arena->stats.nmadvise; 2270 astats->purged += arena->stats.purged; 2271 astats->allocated_large += arena->stats.allocated_large; 2272 astats->nmalloc_large += arena->stats.nmalloc_large; 2273 astats->ndalloc_large += arena->stats.ndalloc_large; 2274 astats->nrequests_large += arena->stats.nrequests_large; 2275 2276 for (i = 0; i < nlclasses; i++) { 2277 lstats[i].nmalloc += arena->stats.lstats[i].nmalloc; 2278 lstats[i].ndalloc += arena->stats.lstats[i].ndalloc; 2279 lstats[i].nrequests += arena->stats.lstats[i].nrequests; 2280 lstats[i].curruns += arena->stats.lstats[i].curruns; 2281 } 2282 malloc_mutex_unlock(&arena->lock); 2283 2284 for (i = 0; i < NBINS; i++) { 2285 arena_bin_t *bin = &arena->bins[i]; 2286 2287 malloc_mutex_lock(&bin->lock); 2288 bstats[i].allocated += bin->stats.allocated; 2289 bstats[i].nmalloc += bin->stats.nmalloc; 2290 bstats[i].ndalloc += bin->stats.ndalloc; 2291 bstats[i].nrequests += bin->stats.nrequests; 2292 if (config_tcache) { 2293 bstats[i].nfills += bin->stats.nfills; 2294 bstats[i].nflushes += bin->stats.nflushes; 2295 } 2296 bstats[i].nruns += bin->stats.nruns; 2297 bstats[i].reruns += bin->stats.reruns; 2298 bstats[i].curruns += bin->stats.curruns; 2299 malloc_mutex_unlock(&bin->lock); 2300 } 2301} 2302 2303bool 2304arena_new(arena_t *arena, unsigned ind) 2305{ 2306 unsigned i; 2307 arena_bin_t *bin; 2308 2309 arena->ind = ind; 2310 arena->nthreads = 0; 2311 2312 if (malloc_mutex_init(&arena->lock)) 2313 return (true); 2314 2315 if (config_stats) { 2316 memset(&arena->stats, 0, sizeof(arena_stats_t)); 2317 arena->stats.lstats = 2318 (malloc_large_stats_t *)base_alloc(nlclasses * 2319 sizeof(malloc_large_stats_t)); 2320 if (arena->stats.lstats == NULL) 2321 return (true); 2322 memset(arena->stats.lstats, 0, nlclasses * 2323 sizeof(malloc_large_stats_t)); 2324 if (config_tcache) 2325 ql_new(&arena->tcache_ql); 2326 } 2327 2328 if (config_prof) 2329 arena->prof_accumbytes = 0; 2330 2331 arena->dss_prec = chunk_dss_prec_get(); 2332 2333 /* Initialize chunks. */ 2334 arena_chunk_dirty_new(&arena->chunks_dirty); 2335 arena->spare = NULL; 2336 2337 arena->nactive = 0; 2338 arena->ndirty = 0; 2339 arena->npurgatory = 0; 2340 2341 arena_avail_tree_new(&arena->runs_avail); 2342 2343 /* Initialize bins. */ 2344 for (i = 0; i < NBINS; i++) { 2345 bin = &arena->bins[i]; 2346 if (malloc_mutex_init(&bin->lock)) 2347 return (true); 2348 bin->runcur = NULL; 2349 arena_run_tree_new(&bin->runs); 2350 if (config_stats) 2351 memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 2352 } 2353 2354 return (false); 2355} 2356 2357/* 2358 * Calculate bin_info->run_size such that it meets the following constraints: 2359 * 2360 * *) bin_info->run_size >= min_run_size 2361 * *) bin_info->run_size <= arena_maxclass 2362 * *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed). 2363 * *) bin_info->nregs <= RUN_MAXREGS 2364 * 2365 * bin_info->nregs, bin_info->bitmap_offset, and bin_info->reg0_offset are also 2366 * calculated here, since these settings are all interdependent. 2367 */ 2368static size_t 2369bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) 2370{ 2371 size_t pad_size; 2372 size_t try_run_size, good_run_size; 2373 uint32_t try_nregs, good_nregs; 2374 uint32_t try_hdr_size, good_hdr_size; 2375 uint32_t try_bitmap_offset, good_bitmap_offset; 2376 uint32_t try_ctx0_offset, good_ctx0_offset; 2377 uint32_t try_redzone0_offset, good_redzone0_offset; 2378 2379 assert(min_run_size >= PAGE); 2380 assert(min_run_size <= arena_maxclass); 2381 2382 /* 2383 * Determine redzone size based on minimum alignment and minimum 2384 * redzone size. Add padding to the end of the run if it is needed to 2385 * align the regions. The padding allows each redzone to be half the 2386 * minimum alignment; without the padding, each redzone would have to 2387 * be twice as large in order to maintain alignment. 2388 */ 2389 if (config_fill && opt_redzone) { 2390 size_t align_min = ZU(1) << (ffs(bin_info->reg_size) - 1); 2391 if (align_min <= REDZONE_MINSIZE) { 2392 bin_info->redzone_size = REDZONE_MINSIZE; 2393 pad_size = 0; 2394 } else { 2395 bin_info->redzone_size = align_min >> 1; 2396 pad_size = bin_info->redzone_size; 2397 } 2398 } else { 2399 bin_info->redzone_size = 0; 2400 pad_size = 0; 2401 } 2402 bin_info->reg_interval = bin_info->reg_size + 2403 (bin_info->redzone_size << 1); 2404 2405 /* 2406 * Calculate known-valid settings before entering the run_size 2407 * expansion loop, so that the first part of the loop always copies 2408 * valid settings. 2409 * 2410 * The do..while loop iteratively reduces the number of regions until 2411 * the run header and the regions no longer overlap. A closed formula 2412 * would be quite messy, since there is an interdependency between the 2413 * header's mask length and the number of regions. 2414 */ 2415 try_run_size = min_run_size; 2416 try_nregs = ((try_run_size - sizeof(arena_run_t)) / 2417 bin_info->reg_interval) 2418 + 1; /* Counter-act try_nregs-- in loop. */ 2419 if (try_nregs > RUN_MAXREGS) { 2420 try_nregs = RUN_MAXREGS 2421 + 1; /* Counter-act try_nregs-- in loop. */ 2422 } 2423 do { 2424 try_nregs--; 2425 try_hdr_size = sizeof(arena_run_t); 2426 /* Pad to a long boundary. */ 2427 try_hdr_size = LONG_CEILING(try_hdr_size); 2428 try_bitmap_offset = try_hdr_size; 2429 /* Add space for bitmap. */ 2430 try_hdr_size += bitmap_size(try_nregs); 2431 if (config_prof && opt_prof && prof_promote == false) { 2432 /* Pad to a quantum boundary. */ 2433 try_hdr_size = QUANTUM_CEILING(try_hdr_size); 2434 try_ctx0_offset = try_hdr_size; 2435 /* Add space for one (prof_ctx_t *) per region. */ 2436 try_hdr_size += try_nregs * sizeof(prof_ctx_t *); 2437 } else 2438 try_ctx0_offset = 0; 2439 try_redzone0_offset = try_run_size - (try_nregs * 2440 bin_info->reg_interval) - pad_size; 2441 } while (try_hdr_size > try_redzone0_offset); 2442 2443 /* run_size expansion loop. */ 2444 do { 2445 /* 2446 * Copy valid settings before trying more aggressive settings. 2447 */ 2448 good_run_size = try_run_size; 2449 good_nregs = try_nregs; 2450 good_hdr_size = try_hdr_size; 2451 good_bitmap_offset = try_bitmap_offset; 2452 good_ctx0_offset = try_ctx0_offset; 2453 good_redzone0_offset = try_redzone0_offset; 2454 2455 /* Try more aggressive settings. */ 2456 try_run_size += PAGE; 2457 try_nregs = ((try_run_size - sizeof(arena_run_t) - pad_size) / 2458 bin_info->reg_interval) 2459 + 1; /* Counter-act try_nregs-- in loop. */ 2460 if (try_nregs > RUN_MAXREGS) { 2461 try_nregs = RUN_MAXREGS 2462 + 1; /* Counter-act try_nregs-- in loop. */ 2463 } 2464 do { 2465 try_nregs--; 2466 try_hdr_size = sizeof(arena_run_t); 2467 /* Pad to a long boundary. */ 2468 try_hdr_size = LONG_CEILING(try_hdr_size); 2469 try_bitmap_offset = try_hdr_size; 2470 /* Add space for bitmap. */ 2471 try_hdr_size += bitmap_size(try_nregs); 2472 if (config_prof && opt_prof && prof_promote == false) { 2473 /* Pad to a quantum boundary. */ 2474 try_hdr_size = QUANTUM_CEILING(try_hdr_size); 2475 try_ctx0_offset = try_hdr_size; 2476 /* 2477 * Add space for one (prof_ctx_t *) per region. 2478 */ 2479 try_hdr_size += try_nregs * 2480 sizeof(prof_ctx_t *); 2481 } 2482 try_redzone0_offset = try_run_size - (try_nregs * 2483 bin_info->reg_interval) - pad_size; 2484 } while (try_hdr_size > try_redzone0_offset); 2485 } while (try_run_size <= arena_maxclass 2486 && RUN_MAX_OVRHD * (bin_info->reg_interval << 3) > 2487 RUN_MAX_OVRHD_RELAX 2488 && (try_redzone0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size 2489 && try_nregs < RUN_MAXREGS); 2490 2491 assert(good_hdr_size <= good_redzone0_offset); 2492 2493 /* Copy final settings. */ 2494 bin_info->run_size = good_run_size; 2495 bin_info->nregs = good_nregs; 2496 bin_info->bitmap_offset = good_bitmap_offset; 2497 bin_info->ctx0_offset = good_ctx0_offset; 2498 bin_info->reg0_offset = good_redzone0_offset + bin_info->redzone_size; 2499 2500 assert(bin_info->reg0_offset - bin_info->redzone_size + (bin_info->nregs 2501 * bin_info->reg_interval) + pad_size == bin_info->run_size); 2502 2503 return (good_run_size); 2504} 2505 2506static void 2507bin_info_init(void) 2508{ 2509 arena_bin_info_t *bin_info; 2510 size_t prev_run_size = PAGE; 2511 2512#define SIZE_CLASS(bin, delta, size) \ 2513 bin_info = &arena_bin_info[bin]; \ 2514 bin_info->reg_size = size; \ 2515 prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size);\ 2516 bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); 2517 SIZE_CLASSES 2518#undef SIZE_CLASS 2519} 2520 2521void 2522arena_boot(void) 2523{ 2524 size_t header_size; 2525 unsigned i; 2526 2527 /* 2528 * Compute the header size such that it is large enough to contain the 2529 * page map. The page map is biased to omit entries for the header 2530 * itself, so some iteration is necessary to compute the map bias. 2531 * 2532 * 1) Compute safe header_size and map_bias values that include enough 2533 * space for an unbiased page map. 2534 * 2) Refine map_bias based on (1) to omit the header pages in the page 2535 * map. The resulting map_bias may be one too small. 2536 * 3) Refine map_bias based on (2). The result will be >= the result 2537 * from (2), and will always be correct. 2538 */ 2539 map_bias = 0; 2540 for (i = 0; i < 3; i++) { 2541 header_size = offsetof(arena_chunk_t, map) + 2542 (sizeof(arena_chunk_map_t) * (chunk_npages-map_bias)); 2543 map_bias = (header_size >> LG_PAGE) + ((header_size & PAGE_MASK) 2544 != 0); 2545 } 2546 assert(map_bias > 0); 2547 2548 arena_maxclass = chunksize - (map_bias << LG_PAGE); 2549 2550 bin_info_init(); 2551} 2552 2553void 2554arena_prefork(arena_t *arena) 2555{ 2556 unsigned i; 2557 2558 malloc_mutex_prefork(&arena->lock); 2559 for (i = 0; i < NBINS; i++) 2560 malloc_mutex_prefork(&arena->bins[i].lock); 2561} 2562 2563void 2564arena_postfork_parent(arena_t *arena) 2565{ 2566 unsigned i; 2567 2568 for (i = 0; i < NBINS; i++) 2569 malloc_mutex_postfork_parent(&arena->bins[i].lock); 2570 malloc_mutex_postfork_parent(&arena->lock); 2571} 2572 2573void 2574arena_postfork_child(arena_t *arena) 2575{ 2576 unsigned i; 2577 2578 for (i = 0; i < NBINS; i++) 2579 malloc_mutex_postfork_child(&arena->bins[i].lock); 2580 malloc_mutex_postfork_child(&arena->lock); 2581} 2582