arena.c revision 021136ce4db79f50031a1fd5dd751891888fbc7b
1#define JEMALLOC_ARENA_C_ 2#include "jemalloc/internal/jemalloc_internal.h" 3 4/******************************************************************************/ 5/* Data. */ 6 7ssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; 8arena_bin_info_t arena_bin_info[NBINS]; 9 10JEMALLOC_ALIGNED(CACHELINE) 11const uint32_t small_bin2size[NBINS] = { 12#define SIZE_CLASS(bin, delta, size) \ 13 size, 14 SIZE_CLASSES 15#undef SIZE_CLASS 16}; 17 18JEMALLOC_ALIGNED(CACHELINE) 19const uint8_t small_size2bin[] = { 20#define S2B_8(i) i, 21#define S2B_16(i) S2B_8(i) S2B_8(i) 22#define S2B_32(i) S2B_16(i) S2B_16(i) 23#define S2B_64(i) S2B_32(i) S2B_32(i) 24#define S2B_128(i) S2B_64(i) S2B_64(i) 25#define S2B_256(i) S2B_128(i) S2B_128(i) 26#define S2B_512(i) S2B_256(i) S2B_256(i) 27#define S2B_1024(i) S2B_512(i) S2B_512(i) 28#define S2B_2048(i) S2B_1024(i) S2B_1024(i) 29#define S2B_4096(i) S2B_2048(i) S2B_2048(i) 30#define S2B_8192(i) S2B_4096(i) S2B_4096(i) 31#define SIZE_CLASS(bin, delta, size) \ 32 S2B_##delta(bin) 33 SIZE_CLASSES 34#undef S2B_8 35#undef S2B_16 36#undef S2B_32 37#undef S2B_64 38#undef S2B_128 39#undef S2B_256 40#undef S2B_512 41#undef S2B_1024 42#undef S2B_2048 43#undef S2B_4096 44#undef S2B_8192 45#undef SIZE_CLASS 46}; 47 48/******************************************************************************/ 49/* 50 * Function prototypes for static functions that are referenced prior to 51 * definition. 52 */ 53 54static void arena_purge(arena_t *arena, bool all); 55static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, 56 bool cleaned); 57static void arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, 58 arena_run_t *run, arena_bin_t *bin); 59static void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, 60 arena_run_t *run, arena_bin_t *bin); 61 62/******************************************************************************/ 63 64JEMALLOC_INLINE_C size_t 65arena_mapelm_to_pageind(arena_chunk_map_t *mapelm) 66{ 67 uintptr_t map_offset = 68 CHUNK_ADDR2OFFSET(mapelm) - offsetof(arena_chunk_t, map); 69 70 return ((map_offset / sizeof(arena_chunk_map_t)) + map_bias); 71} 72 73JEMALLOC_INLINE_C size_t 74arena_mapelm_to_bits(arena_chunk_map_t *mapelm) 75{ 76 77 return (mapelm->bits); 78} 79 80static inline int 81arena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) 82{ 83 uintptr_t a_mapelm = (uintptr_t)a; 84 uintptr_t b_mapelm = (uintptr_t)b; 85 86 assert(a != NULL); 87 assert(b != NULL); 88 89 return ((a_mapelm > b_mapelm) - (a_mapelm < b_mapelm)); 90} 91 92/* Generate red-black tree functions. */ 93rb_gen(static UNUSED, arena_run_tree_, arena_run_tree_t, arena_chunk_map_t, 94 u.rb_link, arena_run_comp) 95 96static inline int 97arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) 98{ 99 int ret; 100 size_t a_size; 101 size_t b_size = arena_mapelm_to_bits(b) & ~PAGE_MASK; 102 uintptr_t a_mapelm = (uintptr_t)a; 103 uintptr_t b_mapelm = (uintptr_t)b; 104 105 if (a_mapelm & CHUNK_MAP_KEY) 106 a_size = a_mapelm & ~PAGE_MASK; 107 else 108 a_size = arena_mapelm_to_bits(a) & ~PAGE_MASK; 109 110 ret = (a_size > b_size) - (a_size < b_size); 111 if (ret == 0 && (!(a_mapelm & CHUNK_MAP_KEY))) 112 ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm); 113 114 return (ret); 115} 116 117/* Generate red-black tree functions. */ 118rb_gen(static UNUSED, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t, 119 u.rb_link, arena_avail_comp) 120 121static inline int 122arena_chunk_dirty_comp(arena_chunk_t *a, arena_chunk_t *b) 123{ 124 125 assert(a != NULL); 126 assert(b != NULL); 127 128 /* 129 * Short-circuit for self comparison. The following comparison code 130 * would come to the same result, but at the cost of executing the slow 131 * path. 132 */ 133 if (a == b) 134 return (0); 135 136 /* 137 * Order such that chunks with higher fragmentation are "less than" 138 * those with lower fragmentation -- purging order is from "least" to 139 * "greatest". Fragmentation is measured as: 140 * 141 * mean current avail run size 142 * -------------------------------- 143 * mean defragmented avail run size 144 * 145 * navail 146 * ----------- 147 * nruns_avail nruns_avail-nruns_adjac 148 * = ========================= = ----------------------- 149 * navail nruns_avail 150 * ----------------------- 151 * nruns_avail-nruns_adjac 152 * 153 * The following code multiplies away the denominator prior to 154 * comparison, in order to avoid division. 155 * 156 */ 157 { 158 size_t a_val = (a->nruns_avail - a->nruns_adjac) * 159 b->nruns_avail; 160 size_t b_val = (b->nruns_avail - b->nruns_adjac) * 161 a->nruns_avail; 162 163 if (a_val < b_val) 164 return (1); 165 if (a_val > b_val) 166 return (-1); 167 } 168 /* 169 * Break ties by chunk address. For fragmented chunks, report lower 170 * addresses as "lower", so that fragmentation reduction happens first 171 * at lower addresses. However, use the opposite ordering for 172 * unfragmented chunks, in order to increase the chances of 173 * re-allocating dirty runs. 174 */ 175 { 176 uintptr_t a_chunk = (uintptr_t)a; 177 uintptr_t b_chunk = (uintptr_t)b; 178 int ret = ((a_chunk > b_chunk) - (a_chunk < b_chunk)); 179 if (a->nruns_adjac == 0) { 180 assert(b->nruns_adjac == 0); 181 ret = -ret; 182 } 183 return (ret); 184 } 185} 186 187/* Generate red-black tree functions. */ 188rb_gen(static UNUSED, arena_chunk_dirty_, arena_chunk_tree_t, arena_chunk_t, 189 dirty_link, arena_chunk_dirty_comp) 190 191static inline bool 192arena_avail_adjac_pred(arena_chunk_t *chunk, size_t pageind) 193{ 194 bool ret; 195 196 if (pageind-1 < map_bias) 197 ret = false; 198 else { 199 ret = (arena_mapbits_allocated_get(chunk, pageind-1) == 0); 200 assert(ret == false || arena_mapbits_dirty_get(chunk, 201 pageind-1) != arena_mapbits_dirty_get(chunk, pageind)); 202 } 203 return (ret); 204} 205 206static inline bool 207arena_avail_adjac_succ(arena_chunk_t *chunk, size_t pageind, size_t npages) 208{ 209 bool ret; 210 211 if (pageind+npages == chunk_npages) 212 ret = false; 213 else { 214 assert(pageind+npages < chunk_npages); 215 ret = (arena_mapbits_allocated_get(chunk, pageind+npages) == 0); 216 assert(ret == false || arena_mapbits_dirty_get(chunk, pageind) 217 != arena_mapbits_dirty_get(chunk, pageind+npages)); 218 } 219 return (ret); 220} 221 222static inline bool 223arena_avail_adjac(arena_chunk_t *chunk, size_t pageind, size_t npages) 224{ 225 226 return (arena_avail_adjac_pred(chunk, pageind) || 227 arena_avail_adjac_succ(chunk, pageind, npages)); 228} 229 230static void 231arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 232 size_t npages, bool maybe_adjac_pred, bool maybe_adjac_succ) 233{ 234 235 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 236 LG_PAGE)); 237 238 /* 239 * chunks_dirty is keyed by nruns_{avail,adjac}, so the chunk must be 240 * removed and reinserted even if the run to be inserted is clean. 241 */ 242 if (chunk->ndirty != 0) 243 arena_chunk_dirty_remove(&arena->chunks_dirty, chunk); 244 245 if (maybe_adjac_pred && arena_avail_adjac_pred(chunk, pageind)) 246 chunk->nruns_adjac++; 247 if (maybe_adjac_succ && arena_avail_adjac_succ(chunk, pageind, npages)) 248 chunk->nruns_adjac++; 249 chunk->nruns_avail++; 250 assert(chunk->nruns_avail > chunk->nruns_adjac); 251 252 if (arena_mapbits_dirty_get(chunk, pageind) != 0) { 253 arena->ndirty += npages; 254 chunk->ndirty += npages; 255 } 256 if (chunk->ndirty != 0) 257 arena_chunk_dirty_insert(&arena->chunks_dirty, chunk); 258 259 arena_avail_tree_insert(&arena->runs_avail, arena_mapp_get(chunk, 260 pageind)); 261} 262 263static void 264arena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 265 size_t npages, bool maybe_adjac_pred, bool maybe_adjac_succ) 266{ 267 268 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 269 LG_PAGE)); 270 271 /* 272 * chunks_dirty is keyed by nruns_{avail,adjac}, so the chunk must be 273 * removed and reinserted even if the run to be removed is clean. 274 */ 275 if (chunk->ndirty != 0) 276 arena_chunk_dirty_remove(&arena->chunks_dirty, chunk); 277 278 if (maybe_adjac_pred && arena_avail_adjac_pred(chunk, pageind)) 279 chunk->nruns_adjac--; 280 if (maybe_adjac_succ && arena_avail_adjac_succ(chunk, pageind, npages)) 281 chunk->nruns_adjac--; 282 chunk->nruns_avail--; 283 assert(chunk->nruns_avail > chunk->nruns_adjac || (chunk->nruns_avail 284 == 0 && chunk->nruns_adjac == 0)); 285 286 if (arena_mapbits_dirty_get(chunk, pageind) != 0) { 287 arena->ndirty -= npages; 288 chunk->ndirty -= npages; 289 } 290 if (chunk->ndirty != 0) 291 arena_chunk_dirty_insert(&arena->chunks_dirty, chunk); 292 293 arena_avail_tree_remove(&arena->runs_avail, arena_mapp_get(chunk, 294 pageind)); 295} 296 297static inline void * 298arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) 299{ 300 void *ret; 301 unsigned regind; 302 bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 303 (uintptr_t)bin_info->bitmap_offset); 304 305 assert(run->nfree > 0); 306 assert(bitmap_full(bitmap, &bin_info->bitmap_info) == false); 307 308 regind = bitmap_sfu(bitmap, &bin_info->bitmap_info); 309 ret = (void *)((uintptr_t)run + (uintptr_t)bin_info->reg0_offset + 310 (uintptr_t)(bin_info->reg_interval * regind)); 311 run->nfree--; 312 if (regind == run->nextind) 313 run->nextind++; 314 assert(regind < run->nextind); 315 return (ret); 316} 317 318static inline void 319arena_run_reg_dalloc(arena_run_t *run, void *ptr) 320{ 321 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 322 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 323 size_t mapbits = arena_mapbits_get(chunk, pageind); 324 size_t binind = arena_ptr_small_binind_get(ptr, mapbits); 325 arena_bin_info_t *bin_info = &arena_bin_info[binind]; 326 unsigned regind = arena_run_regind(run, bin_info, ptr); 327 bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 328 (uintptr_t)bin_info->bitmap_offset); 329 330 assert(run->nfree < bin_info->nregs); 331 /* Freeing an interior pointer can cause assertion failure. */ 332 assert(((uintptr_t)ptr - ((uintptr_t)run + 333 (uintptr_t)bin_info->reg0_offset)) % 334 (uintptr_t)bin_info->reg_interval == 0); 335 assert((uintptr_t)ptr >= (uintptr_t)run + 336 (uintptr_t)bin_info->reg0_offset); 337 /* Freeing an unallocated pointer can cause assertion failure. */ 338 assert(bitmap_get(bitmap, &bin_info->bitmap_info, regind)); 339 340 bitmap_unset(bitmap, &bin_info->bitmap_info, regind); 341 run->nfree++; 342} 343 344static inline void 345arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages) 346{ 347 348 VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + (run_ind << 349 LG_PAGE)), (npages << LG_PAGE)); 350 memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0, 351 (npages << LG_PAGE)); 352} 353 354static inline void 355arena_run_page_mark_zeroed(arena_chunk_t *chunk, size_t run_ind) 356{ 357 358 VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind << 359 LG_PAGE)), PAGE); 360} 361 362static inline void 363arena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) 364{ 365 size_t i; 366 UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE)); 367 368 arena_run_page_mark_zeroed(chunk, run_ind); 369 for (i = 0; i < PAGE / sizeof(size_t); i++) 370 assert(p[i] == 0); 371} 372 373static void 374arena_cactive_update(arena_t *arena, size_t add_pages, size_t sub_pages) 375{ 376 377 if (config_stats) { 378 ssize_t cactive_diff = CHUNK_CEILING((arena->nactive + 379 add_pages) << LG_PAGE) - CHUNK_CEILING((arena->nactive - 380 sub_pages) << LG_PAGE); 381 if (cactive_diff != 0) 382 stats_cactive_add(cactive_diff); 383 } 384} 385 386static void 387arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, 388 size_t flag_dirty, size_t need_pages) 389{ 390 size_t total_pages, rem_pages; 391 392 total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >> 393 LG_PAGE; 394 assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) == 395 flag_dirty); 396 assert(need_pages <= total_pages); 397 rem_pages = total_pages - need_pages; 398 399 arena_avail_remove(arena, chunk, run_ind, total_pages, true, true); 400 arena_cactive_update(arena, need_pages, 0); 401 arena->nactive += need_pages; 402 403 /* Keep track of trailing unused pages for later use. */ 404 if (rem_pages > 0) { 405 if (flag_dirty != 0) { 406 arena_mapbits_unallocated_set(chunk, 407 run_ind+need_pages, (rem_pages << LG_PAGE), 408 flag_dirty); 409 arena_mapbits_unallocated_set(chunk, 410 run_ind+total_pages-1, (rem_pages << LG_PAGE), 411 flag_dirty); 412 } else { 413 arena_mapbits_unallocated_set(chunk, run_ind+need_pages, 414 (rem_pages << LG_PAGE), 415 arena_mapbits_unzeroed_get(chunk, 416 run_ind+need_pages)); 417 arena_mapbits_unallocated_set(chunk, 418 run_ind+total_pages-1, (rem_pages << LG_PAGE), 419 arena_mapbits_unzeroed_get(chunk, 420 run_ind+total_pages-1)); 421 } 422 arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages, 423 false, true); 424 } 425} 426 427static void 428arena_run_split_large_helper(arena_t *arena, arena_run_t *run, size_t size, 429 bool remove, bool zero) 430{ 431 arena_chunk_t *chunk; 432 size_t flag_dirty, run_ind, need_pages, i; 433 434 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 435 run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 436 flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); 437 need_pages = (size >> LG_PAGE); 438 assert(need_pages > 0); 439 440 if (remove) { 441 arena_run_split_remove(arena, chunk, run_ind, flag_dirty, 442 need_pages); 443 } 444 445 if (zero) { 446 if (flag_dirty == 0) { 447 /* 448 * The run is clean, so some pages may be zeroed (i.e. 449 * never before touched). 450 */ 451 for (i = 0; i < need_pages; i++) { 452 if (arena_mapbits_unzeroed_get(chunk, run_ind+i) 453 != 0) 454 arena_run_zero(chunk, run_ind+i, 1); 455 else if (config_debug) { 456 arena_run_page_validate_zeroed(chunk, 457 run_ind+i); 458 } else { 459 arena_run_page_mark_zeroed(chunk, 460 run_ind+i); 461 } 462 } 463 } else { 464 /* The run is dirty, so all pages must be zeroed. */ 465 arena_run_zero(chunk, run_ind, need_pages); 466 } 467 } else { 468 VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + 469 (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); 470 } 471 472 /* 473 * Set the last element first, in case the run only contains one page 474 * (i.e. both statements set the same element). 475 */ 476 arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0, flag_dirty); 477 arena_mapbits_large_set(chunk, run_ind, size, flag_dirty); 478} 479 480static void 481arena_run_split_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) 482{ 483 484 arena_run_split_large_helper(arena, run, size, true, zero); 485} 486 487static void 488arena_run_init_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) 489{ 490 491 arena_run_split_large_helper(arena, run, size, false, zero); 492} 493 494static void 495arena_run_split_small(arena_t *arena, arena_run_t *run, size_t size, 496 size_t binind) 497{ 498 arena_chunk_t *chunk; 499 size_t flag_dirty, run_ind, need_pages, i; 500 501 assert(binind != BININD_INVALID); 502 503 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 504 run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 505 flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); 506 need_pages = (size >> LG_PAGE); 507 assert(need_pages > 0); 508 509 arena_run_split_remove(arena, chunk, run_ind, flag_dirty, need_pages); 510 511 /* 512 * Propagate the dirty and unzeroed flags to the allocated small run, 513 * so that arena_dalloc_bin_run() has the ability to conditionally trim 514 * clean pages. 515 */ 516 arena_mapbits_small_set(chunk, run_ind, 0, binind, flag_dirty); 517 /* 518 * The first page will always be dirtied during small run 519 * initialization, so a validation failure here would not actually 520 * cause an observable failure. 521 */ 522 if (config_debug && flag_dirty == 0 && arena_mapbits_unzeroed_get(chunk, 523 run_ind) == 0) 524 arena_run_page_validate_zeroed(chunk, run_ind); 525 for (i = 1; i < need_pages - 1; i++) { 526 arena_mapbits_small_set(chunk, run_ind+i, i, binind, 0); 527 if (config_debug && flag_dirty == 0 && 528 arena_mapbits_unzeroed_get(chunk, run_ind+i) == 0) 529 arena_run_page_validate_zeroed(chunk, run_ind+i); 530 } 531 arena_mapbits_small_set(chunk, run_ind+need_pages-1, need_pages-1, 532 binind, flag_dirty); 533 if (config_debug && flag_dirty == 0 && arena_mapbits_unzeroed_get(chunk, 534 run_ind+need_pages-1) == 0) 535 arena_run_page_validate_zeroed(chunk, run_ind+need_pages-1); 536 VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + 537 (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); 538} 539 540static arena_chunk_t * 541arena_chunk_init_spare(arena_t *arena) 542{ 543 arena_chunk_t *chunk; 544 545 assert(arena->spare != NULL); 546 547 chunk = arena->spare; 548 arena->spare = NULL; 549 550 assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); 551 assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); 552 assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == 553 arena_maxclass); 554 assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == 555 arena_maxclass); 556 assert(arena_mapbits_dirty_get(chunk, map_bias) == 557 arena_mapbits_dirty_get(chunk, chunk_npages-1)); 558 559 return (chunk); 560} 561 562static arena_chunk_t * 563arena_chunk_init_hard(arena_t *arena) 564{ 565 arena_chunk_t *chunk; 566 bool zero; 567 size_t unzeroed, i; 568 569 assert(arena->spare == NULL); 570 571 zero = false; 572 malloc_mutex_unlock(&arena->lock); 573 chunk = (arena_chunk_t *)chunk_alloc(chunksize, chunksize, false, 574 &zero, arena->dss_prec); 575 malloc_mutex_lock(&arena->lock); 576 if (chunk == NULL) 577 return (NULL); 578 if (config_stats) 579 arena->stats.mapped += chunksize; 580 581 chunk->arena = arena; 582 583 /* 584 * Claim that no pages are in use, since the header is merely overhead. 585 */ 586 chunk->ndirty = 0; 587 588 chunk->nruns_avail = 0; 589 chunk->nruns_adjac = 0; 590 591 /* 592 * Initialize the map to contain one maximal free untouched run. Mark 593 * the pages as zeroed iff chunk_alloc() returned a zeroed chunk. 594 */ 595 unzeroed = zero ? 0 : CHUNK_MAP_UNZEROED; 596 arena_mapbits_unallocated_set(chunk, map_bias, arena_maxclass, 597 unzeroed); 598 /* 599 * There is no need to initialize the internal page map entries unless 600 * the chunk is not zeroed. 601 */ 602 if (zero == false) { 603 VALGRIND_MAKE_MEM_UNDEFINED((void *)arena_mapp_get(chunk, 604 map_bias+1), (size_t)((uintptr_t) arena_mapp_get(chunk, 605 chunk_npages-1) - (uintptr_t)arena_mapp_get(chunk, 606 map_bias+1))); 607 for (i = map_bias+1; i < chunk_npages-1; i++) 608 arena_mapbits_unzeroed_set(chunk, i, unzeroed); 609 } else { 610 VALGRIND_MAKE_MEM_DEFINED((void *)arena_mapp_get(chunk, 611 map_bias+1), (size_t)((uintptr_t) arena_mapp_get(chunk, 612 chunk_npages-1) - (uintptr_t)arena_mapp_get(chunk, 613 map_bias+1))); 614 if (config_debug) { 615 for (i = map_bias+1; i < chunk_npages-1; i++) { 616 assert(arena_mapbits_unzeroed_get(chunk, i) == 617 unzeroed); 618 } 619 } 620 } 621 arena_mapbits_unallocated_set(chunk, chunk_npages-1, arena_maxclass, 622 unzeroed); 623 624 return (chunk); 625} 626 627static arena_chunk_t * 628arena_chunk_alloc(arena_t *arena) 629{ 630 arena_chunk_t *chunk; 631 632 if (arena->spare != NULL) 633 chunk = arena_chunk_init_spare(arena); 634 else { 635 chunk = arena_chunk_init_hard(arena); 636 if (chunk == NULL) 637 return (NULL); 638 } 639 640 /* Insert the run into the runs_avail tree. */ 641 arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias, 642 false, false); 643 644 return (chunk); 645} 646 647static void 648arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk) 649{ 650 assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); 651 assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); 652 assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == 653 arena_maxclass); 654 assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == 655 arena_maxclass); 656 assert(arena_mapbits_dirty_get(chunk, map_bias) == 657 arena_mapbits_dirty_get(chunk, chunk_npages-1)); 658 659 /* 660 * Remove run from the runs_avail tree, so that the arena does not use 661 * it. 662 */ 663 arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias, 664 false, false); 665 666 if (arena->spare != NULL) { 667 arena_chunk_t *spare = arena->spare; 668 669 arena->spare = chunk; 670 malloc_mutex_unlock(&arena->lock); 671 chunk_dealloc((void *)spare, chunksize, true); 672 malloc_mutex_lock(&arena->lock); 673 if (config_stats) 674 arena->stats.mapped -= chunksize; 675 } else 676 arena->spare = chunk; 677} 678 679static arena_run_t * 680arena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero) 681{ 682 arena_run_t *run; 683 arena_chunk_map_t *mapelm; 684 arena_chunk_map_t *key; 685 686 key = (arena_chunk_map_t *)(size | CHUNK_MAP_KEY); 687 mapelm = arena_avail_tree_nsearch(&arena->runs_avail, key); 688 if (mapelm != NULL) { 689 arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); 690 size_t pageind = arena_mapelm_to_pageind(mapelm); 691 692 run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << 693 LG_PAGE)); 694 arena_run_split_large(arena, run, size, zero); 695 return (run); 696 } 697 698 return (NULL); 699} 700 701static arena_run_t * 702arena_run_alloc_large(arena_t *arena, size_t size, bool zero) 703{ 704 arena_chunk_t *chunk; 705 arena_run_t *run; 706 707 assert(size <= arena_maxclass); 708 assert((size & PAGE_MASK) == 0); 709 710 /* Search the arena's chunks for the lowest best fit. */ 711 run = arena_run_alloc_large_helper(arena, size, zero); 712 if (run != NULL) 713 return (run); 714 715 /* 716 * No usable runs. Create a new chunk from which to allocate the run. 717 */ 718 chunk = arena_chunk_alloc(arena); 719 if (chunk != NULL) { 720 run = (arena_run_t *)((uintptr_t)chunk + (map_bias << LG_PAGE)); 721 arena_run_split_large(arena, run, size, zero); 722 return (run); 723 } 724 725 /* 726 * arena_chunk_alloc() failed, but another thread may have made 727 * sufficient memory available while this one dropped arena->lock in 728 * arena_chunk_alloc(), so search one more time. 729 */ 730 return (arena_run_alloc_large_helper(arena, size, zero)); 731} 732 733static arena_run_t * 734arena_run_alloc_small_helper(arena_t *arena, size_t size, size_t binind) 735{ 736 arena_run_t *run; 737 arena_chunk_map_t *mapelm; 738 arena_chunk_map_t *key; 739 740 key = (arena_chunk_map_t *)(size | CHUNK_MAP_KEY); 741 mapelm = arena_avail_tree_nsearch(&arena->runs_avail, key); 742 if (mapelm != NULL) { 743 arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); 744 size_t pageind = arena_mapelm_to_pageind(mapelm); 745 746 run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << 747 LG_PAGE)); 748 arena_run_split_small(arena, run, size, binind); 749 return (run); 750 } 751 752 return (NULL); 753} 754 755static arena_run_t * 756arena_run_alloc_small(arena_t *arena, size_t size, size_t binind) 757{ 758 arena_chunk_t *chunk; 759 arena_run_t *run; 760 761 assert(size <= arena_maxclass); 762 assert((size & PAGE_MASK) == 0); 763 assert(binind != BININD_INVALID); 764 765 /* Search the arena's chunks for the lowest best fit. */ 766 run = arena_run_alloc_small_helper(arena, size, binind); 767 if (run != NULL) 768 return (run); 769 770 /* 771 * No usable runs. Create a new chunk from which to allocate the run. 772 */ 773 chunk = arena_chunk_alloc(arena); 774 if (chunk != NULL) { 775 run = (arena_run_t *)((uintptr_t)chunk + (map_bias << LG_PAGE)); 776 arena_run_split_small(arena, run, size, binind); 777 return (run); 778 } 779 780 /* 781 * arena_chunk_alloc() failed, but another thread may have made 782 * sufficient memory available while this one dropped arena->lock in 783 * arena_chunk_alloc(), so search one more time. 784 */ 785 return (arena_run_alloc_small_helper(arena, size, binind)); 786} 787 788static inline void 789arena_maybe_purge(arena_t *arena) 790{ 791 size_t npurgeable, threshold; 792 793 /* Don't purge if the option is disabled. */ 794 if (opt_lg_dirty_mult < 0) 795 return; 796 /* Don't purge if all dirty pages are already being purged. */ 797 if (arena->ndirty <= arena->npurgatory) 798 return; 799 npurgeable = arena->ndirty - arena->npurgatory; 800 threshold = (arena->nactive >> opt_lg_dirty_mult); 801 /* 802 * Don't purge unless the number of purgeable pages exceeds the 803 * threshold. 804 */ 805 if (npurgeable <= threshold) 806 return; 807 808 arena_purge(arena, false); 809} 810 811static arena_chunk_t * 812chunks_dirty_iter_cb(arena_chunk_tree_t *tree, arena_chunk_t *chunk, void *arg) 813{ 814 size_t *ndirty = (size_t *)arg; 815 816 assert(chunk->ndirty != 0); 817 *ndirty += chunk->ndirty; 818 return (NULL); 819} 820 821static size_t 822arena_compute_npurgatory(arena_t *arena, bool all) 823{ 824 size_t npurgatory, npurgeable; 825 826 /* 827 * Compute the minimum number of pages that this thread should try to 828 * purge. 829 */ 830 npurgeable = arena->ndirty - arena->npurgatory; 831 832 if (all == false) { 833 size_t threshold = (arena->nactive >> opt_lg_dirty_mult); 834 835 npurgatory = npurgeable - threshold; 836 } else 837 npurgatory = npurgeable; 838 839 return (npurgatory); 840} 841 842static void 843arena_chunk_stash_dirty(arena_t *arena, arena_chunk_t *chunk, bool all, 844 arena_chunk_mapelms_t *mapelms) 845{ 846 size_t pageind, npages; 847 848 /* 849 * Temporarily allocate free dirty runs within chunk. If all is false, 850 * only operate on dirty runs that are fragments; otherwise operate on 851 * all dirty runs. 852 */ 853 for (pageind = map_bias; pageind < chunk_npages; pageind += npages) { 854 arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); 855 if (arena_mapbits_allocated_get(chunk, pageind) == 0) { 856 size_t run_size = 857 arena_mapbits_unallocated_size_get(chunk, pageind); 858 859 npages = run_size >> LG_PAGE; 860 assert(pageind + npages <= chunk_npages); 861 assert(arena_mapbits_dirty_get(chunk, pageind) == 862 arena_mapbits_dirty_get(chunk, pageind+npages-1)); 863 864 if (arena_mapbits_dirty_get(chunk, pageind) != 0 && 865 (all || arena_avail_adjac(chunk, pageind, 866 npages))) { 867 arena_run_t *run = (arena_run_t *)((uintptr_t) 868 chunk + (uintptr_t)(pageind << LG_PAGE)); 869 870 arena_run_split_large(arena, run, run_size, 871 false); 872 /* Append to list for later processing. */ 873 ql_elm_new(mapelm, u.ql_link); 874 ql_tail_insert(mapelms, mapelm, u.ql_link); 875 } 876 } else { 877 /* Skip run. */ 878 if (arena_mapbits_large_get(chunk, pageind) != 0) { 879 npages = arena_mapbits_large_size_get(chunk, 880 pageind) >> LG_PAGE; 881 } else { 882 size_t binind; 883 arena_bin_info_t *bin_info; 884 arena_run_t *run = (arena_run_t *)((uintptr_t) 885 chunk + (uintptr_t)(pageind << LG_PAGE)); 886 887 assert(arena_mapbits_small_runind_get(chunk, 888 pageind) == 0); 889 binind = arena_bin_index(arena, run->bin); 890 bin_info = &arena_bin_info[binind]; 891 npages = bin_info->run_size >> LG_PAGE; 892 } 893 } 894 } 895 assert(pageind == chunk_npages); 896 assert(chunk->ndirty == 0 || all == false); 897 assert(chunk->nruns_adjac == 0); 898} 899 900static size_t 901arena_chunk_purge_stashed(arena_t *arena, arena_chunk_t *chunk, 902 arena_chunk_mapelms_t *mapelms) 903{ 904 size_t npurged, pageind, npages, nmadvise; 905 arena_chunk_map_t *mapelm; 906 907 malloc_mutex_unlock(&arena->lock); 908 if (config_stats) 909 nmadvise = 0; 910 npurged = 0; 911 ql_foreach(mapelm, mapelms, u.ql_link) { 912 bool unzeroed; 913 size_t flag_unzeroed, i; 914 915 pageind = arena_mapelm_to_pageind(mapelm); 916 npages = arena_mapbits_large_size_get(chunk, pageind) >> 917 LG_PAGE; 918 assert(pageind + npages <= chunk_npages); 919 unzeroed = pages_purge((void *)((uintptr_t)chunk + (pageind << 920 LG_PAGE)), (npages << LG_PAGE)); 921 flag_unzeroed = unzeroed ? CHUNK_MAP_UNZEROED : 0; 922 /* 923 * Set the unzeroed flag for all pages, now that pages_purge() 924 * has returned whether the pages were zeroed as a side effect 925 * of purging. This chunk map modification is safe even though 926 * the arena mutex isn't currently owned by this thread, 927 * because the run is marked as allocated, thus protecting it 928 * from being modified by any other thread. As long as these 929 * writes don't perturb the first and last elements' 930 * CHUNK_MAP_ALLOCATED bits, behavior is well defined. 931 */ 932 for (i = 0; i < npages; i++) { 933 arena_mapbits_unzeroed_set(chunk, pageind+i, 934 flag_unzeroed); 935 } 936 npurged += npages; 937 if (config_stats) 938 nmadvise++; 939 } 940 malloc_mutex_lock(&arena->lock); 941 if (config_stats) 942 arena->stats.nmadvise += nmadvise; 943 944 return (npurged); 945} 946 947static void 948arena_chunk_unstash_purged(arena_t *arena, arena_chunk_t *chunk, 949 arena_chunk_mapelms_t *mapelms) 950{ 951 arena_chunk_map_t *mapelm; 952 size_t pageind; 953 954 /* Deallocate runs. */ 955 for (mapelm = ql_first(mapelms); mapelm != NULL; 956 mapelm = ql_first(mapelms)) { 957 arena_run_t *run; 958 959 pageind = arena_mapelm_to_pageind(mapelm); 960 run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)(pageind << 961 LG_PAGE)); 962 ql_remove(mapelms, mapelm, u.ql_link); 963 arena_run_dalloc(arena, run, false, true); 964 } 965} 966 967static inline size_t 968arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk, bool all) 969{ 970 size_t npurged; 971 arena_chunk_mapelms_t mapelms; 972 973 ql_new(&mapelms); 974 975 /* 976 * If chunk is the spare, temporarily re-allocate it, 1) so that its 977 * run is reinserted into runs_avail, and 2) so that it cannot be 978 * completely discarded by another thread while arena->lock is dropped 979 * by this thread. Note that the arena_run_dalloc() call will 980 * implicitly deallocate the chunk, so no explicit action is required 981 * in this function to deallocate the chunk. 982 * 983 * Note that once a chunk contains dirty pages, it cannot again contain 984 * a single run unless 1) it is a dirty run, or 2) this function purges 985 * dirty pages and causes the transition to a single clean run. Thus 986 * (chunk == arena->spare) is possible, but it is not possible for 987 * this function to be called on the spare unless it contains a dirty 988 * run. 989 */ 990 if (chunk == arena->spare) { 991 assert(arena_mapbits_dirty_get(chunk, map_bias) != 0); 992 assert(arena_mapbits_dirty_get(chunk, chunk_npages-1) != 0); 993 994 arena_chunk_alloc(arena); 995 } 996 997 if (config_stats) 998 arena->stats.purged += chunk->ndirty; 999 1000 /* 1001 * Operate on all dirty runs if there is no clean/dirty run 1002 * fragmentation. 1003 */ 1004 if (chunk->nruns_adjac == 0) 1005 all = true; 1006 1007 arena_chunk_stash_dirty(arena, chunk, all, &mapelms); 1008 npurged = arena_chunk_purge_stashed(arena, chunk, &mapelms); 1009 arena_chunk_unstash_purged(arena, chunk, &mapelms); 1010 1011 return (npurged); 1012} 1013 1014static void 1015arena_purge(arena_t *arena, bool all) 1016{ 1017 arena_chunk_t *chunk; 1018 size_t npurgatory; 1019 if (config_debug) { 1020 size_t ndirty = 0; 1021 1022 arena_chunk_dirty_iter(&arena->chunks_dirty, NULL, 1023 chunks_dirty_iter_cb, (void *)&ndirty); 1024 assert(ndirty == arena->ndirty); 1025 } 1026 assert(arena->ndirty > arena->npurgatory || all); 1027 assert((arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty - 1028 arena->npurgatory) || all); 1029 1030 if (config_stats) 1031 arena->stats.npurge++; 1032 1033 /* 1034 * Add the minimum number of pages this thread should try to purge to 1035 * arena->npurgatory. This will keep multiple threads from racing to 1036 * reduce ndirty below the threshold. 1037 */ 1038 npurgatory = arena_compute_npurgatory(arena, all); 1039 arena->npurgatory += npurgatory; 1040 1041 while (npurgatory > 0) { 1042 size_t npurgeable, npurged, nunpurged; 1043 1044 /* Get next chunk with dirty pages. */ 1045 chunk = arena_chunk_dirty_first(&arena->chunks_dirty); 1046 if (chunk == NULL) { 1047 /* 1048 * This thread was unable to purge as many pages as 1049 * originally intended, due to races with other threads 1050 * that either did some of the purging work, or re-used 1051 * dirty pages. 1052 */ 1053 arena->npurgatory -= npurgatory; 1054 return; 1055 } 1056 npurgeable = chunk->ndirty; 1057 assert(npurgeable != 0); 1058 1059 if (npurgeable > npurgatory && chunk->nruns_adjac == 0) { 1060 /* 1061 * This thread will purge all the dirty pages in chunk, 1062 * so set npurgatory to reflect this thread's intent to 1063 * purge the pages. This tends to reduce the chances 1064 * of the following scenario: 1065 * 1066 * 1) This thread sets arena->npurgatory such that 1067 * (arena->ndirty - arena->npurgatory) is at the 1068 * threshold. 1069 * 2) This thread drops arena->lock. 1070 * 3) Another thread causes one or more pages to be 1071 * dirtied, and immediately determines that it must 1072 * purge dirty pages. 1073 * 1074 * If this scenario *does* play out, that's okay, 1075 * because all of the purging work being done really 1076 * needs to happen. 1077 */ 1078 arena->npurgatory += npurgeable - npurgatory; 1079 npurgatory = npurgeable; 1080 } 1081 1082 /* 1083 * Keep track of how many pages are purgeable, versus how many 1084 * actually get purged, and adjust counters accordingly. 1085 */ 1086 arena->npurgatory -= npurgeable; 1087 npurgatory -= npurgeable; 1088 npurged = arena_chunk_purge(arena, chunk, all); 1089 nunpurged = npurgeable - npurged; 1090 arena->npurgatory += nunpurged; 1091 npurgatory += nunpurged; 1092 } 1093} 1094 1095void 1096arena_purge_all(arena_t *arena) 1097{ 1098 1099 malloc_mutex_lock(&arena->lock); 1100 arena_purge(arena, true); 1101 malloc_mutex_unlock(&arena->lock); 1102} 1103 1104static void 1105arena_run_coalesce(arena_t *arena, arena_chunk_t *chunk, size_t *p_size, 1106 size_t *p_run_ind, size_t *p_run_pages, size_t flag_dirty) 1107{ 1108 size_t size = *p_size; 1109 size_t run_ind = *p_run_ind; 1110 size_t run_pages = *p_run_pages; 1111 1112 /* Try to coalesce forward. */ 1113 if (run_ind + run_pages < chunk_npages && 1114 arena_mapbits_allocated_get(chunk, run_ind+run_pages) == 0 && 1115 arena_mapbits_dirty_get(chunk, run_ind+run_pages) == flag_dirty) { 1116 size_t nrun_size = arena_mapbits_unallocated_size_get(chunk, 1117 run_ind+run_pages); 1118 size_t nrun_pages = nrun_size >> LG_PAGE; 1119 1120 /* 1121 * Remove successor from runs_avail; the coalesced run is 1122 * inserted later. 1123 */ 1124 assert(arena_mapbits_unallocated_size_get(chunk, 1125 run_ind+run_pages+nrun_pages-1) == nrun_size); 1126 assert(arena_mapbits_dirty_get(chunk, 1127 run_ind+run_pages+nrun_pages-1) == flag_dirty); 1128 arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages, 1129 false, true); 1130 1131 size += nrun_size; 1132 run_pages += nrun_pages; 1133 1134 arena_mapbits_unallocated_size_set(chunk, run_ind, size); 1135 arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, 1136 size); 1137 } 1138 1139 /* Try to coalesce backward. */ 1140 if (run_ind > map_bias && arena_mapbits_allocated_get(chunk, 1141 run_ind-1) == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) == 1142 flag_dirty) { 1143 size_t prun_size = arena_mapbits_unallocated_size_get(chunk, 1144 run_ind-1); 1145 size_t prun_pages = prun_size >> LG_PAGE; 1146 1147 run_ind -= prun_pages; 1148 1149 /* 1150 * Remove predecessor from runs_avail; the coalesced run is 1151 * inserted later. 1152 */ 1153 assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == 1154 prun_size); 1155 assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty); 1156 arena_avail_remove(arena, chunk, run_ind, prun_pages, true, 1157 false); 1158 1159 size += prun_size; 1160 run_pages += prun_pages; 1161 1162 arena_mapbits_unallocated_size_set(chunk, run_ind, size); 1163 arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, 1164 size); 1165 } 1166 1167 *p_size = size; 1168 *p_run_ind = run_ind; 1169 *p_run_pages = run_pages; 1170} 1171 1172static void 1173arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) 1174{ 1175 arena_chunk_t *chunk; 1176 size_t size, run_ind, run_pages, flag_dirty; 1177 1178 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1179 run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 1180 assert(run_ind >= map_bias); 1181 assert(run_ind < chunk_npages); 1182 if (arena_mapbits_large_get(chunk, run_ind) != 0) { 1183 size = arena_mapbits_large_size_get(chunk, run_ind); 1184 assert(size == PAGE || 1185 arena_mapbits_large_size_get(chunk, 1186 run_ind+(size>>LG_PAGE)-1) == 0); 1187 } else { 1188 size_t binind = arena_bin_index(arena, run->bin); 1189 arena_bin_info_t *bin_info = &arena_bin_info[binind]; 1190 size = bin_info->run_size; 1191 } 1192 run_pages = (size >> LG_PAGE); 1193 arena_cactive_update(arena, 0, run_pages); 1194 arena->nactive -= run_pages; 1195 1196 /* 1197 * The run is dirty if the caller claims to have dirtied it, as well as 1198 * if it was already dirty before being allocated and the caller 1199 * doesn't claim to have cleaned it. 1200 */ 1201 assert(arena_mapbits_dirty_get(chunk, run_ind) == 1202 arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); 1203 if (cleaned == false && arena_mapbits_dirty_get(chunk, run_ind) != 0) 1204 dirty = true; 1205 flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; 1206 1207 /* Mark pages as unallocated in the chunk map. */ 1208 if (dirty) { 1209 arena_mapbits_unallocated_set(chunk, run_ind, size, 1210 CHUNK_MAP_DIRTY); 1211 arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, 1212 CHUNK_MAP_DIRTY); 1213 } else { 1214 arena_mapbits_unallocated_set(chunk, run_ind, size, 1215 arena_mapbits_unzeroed_get(chunk, run_ind)); 1216 arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, 1217 arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1)); 1218 } 1219 1220 arena_run_coalesce(arena, chunk, &size, &run_ind, &run_pages, 1221 flag_dirty); 1222 1223 /* Insert into runs_avail, now that coalescing is complete. */ 1224 assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == 1225 arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1)); 1226 assert(arena_mapbits_dirty_get(chunk, run_ind) == 1227 arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); 1228 arena_avail_insert(arena, chunk, run_ind, run_pages, true, true); 1229 1230 /* Deallocate chunk if it is now completely unused. */ 1231 if (size == arena_maxclass) { 1232 assert(run_ind == map_bias); 1233 assert(run_pages == (arena_maxclass >> LG_PAGE)); 1234 arena_chunk_dealloc(arena, chunk); 1235 } 1236 1237 /* 1238 * It is okay to do dirty page processing here even if the chunk was 1239 * deallocated above, since in that case it is the spare. Waiting 1240 * until after possible chunk deallocation to do dirty processing 1241 * allows for an old spare to be fully deallocated, thus decreasing the 1242 * chances of spuriously crossing the dirty page purging threshold. 1243 */ 1244 if (dirty) 1245 arena_maybe_purge(arena); 1246} 1247 1248static void 1249arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1250 size_t oldsize, size_t newsize) 1251{ 1252 size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1253 size_t head_npages = (oldsize - newsize) >> LG_PAGE; 1254 size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); 1255 1256 assert(oldsize > newsize); 1257 1258 /* 1259 * Update the chunk map so that arena_run_dalloc() can treat the 1260 * leading run as separately allocated. Set the last element of each 1261 * run first, in case of single-page runs. 1262 */ 1263 assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); 1264 arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty); 1265 arena_mapbits_large_set(chunk, pageind, oldsize-newsize, flag_dirty); 1266 1267 if (config_debug) { 1268 UNUSED size_t tail_npages = newsize >> LG_PAGE; 1269 assert(arena_mapbits_large_size_get(chunk, 1270 pageind+head_npages+tail_npages-1) == 0); 1271 assert(arena_mapbits_dirty_get(chunk, 1272 pageind+head_npages+tail_npages-1) == flag_dirty); 1273 } 1274 arena_mapbits_large_set(chunk, pageind+head_npages, newsize, 1275 flag_dirty); 1276 1277 arena_run_dalloc(arena, run, false, false); 1278} 1279 1280static void 1281arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1282 size_t oldsize, size_t newsize, bool dirty) 1283{ 1284 size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1285 size_t head_npages = newsize >> LG_PAGE; 1286 size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); 1287 1288 assert(oldsize > newsize); 1289 1290 /* 1291 * Update the chunk map so that arena_run_dalloc() can treat the 1292 * trailing run as separately allocated. Set the last element of each 1293 * run first, in case of single-page runs. 1294 */ 1295 assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); 1296 arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty); 1297 arena_mapbits_large_set(chunk, pageind, newsize, flag_dirty); 1298 1299 if (config_debug) { 1300 UNUSED size_t tail_npages = (oldsize - newsize) >> LG_PAGE; 1301 assert(arena_mapbits_large_size_get(chunk, 1302 pageind+head_npages+tail_npages-1) == 0); 1303 assert(arena_mapbits_dirty_get(chunk, 1304 pageind+head_npages+tail_npages-1) == flag_dirty); 1305 } 1306 arena_mapbits_large_set(chunk, pageind+head_npages, oldsize-newsize, 1307 flag_dirty); 1308 1309 arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize), 1310 dirty, false); 1311} 1312 1313static arena_run_t * 1314arena_bin_runs_first(arena_bin_t *bin) 1315{ 1316 arena_chunk_map_t *mapelm = arena_run_tree_first(&bin->runs); 1317 if (mapelm != NULL) { 1318 arena_chunk_t *chunk; 1319 size_t pageind; 1320 arena_run_t *run; 1321 1322 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); 1323 pageind = arena_mapelm_to_pageind(mapelm); 1324 run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 1325 arena_mapbits_small_runind_get(chunk, pageind)) << 1326 LG_PAGE)); 1327 return (run); 1328 } 1329 1330 return (NULL); 1331} 1332 1333static void 1334arena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run) 1335{ 1336 arena_chunk_t *chunk = CHUNK_ADDR2BASE(run); 1337 size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1338 arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); 1339 1340 assert(arena_run_tree_search(&bin->runs, mapelm) == NULL); 1341 1342 arena_run_tree_insert(&bin->runs, mapelm); 1343} 1344 1345static void 1346arena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run) 1347{ 1348 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1349 size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1350 arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); 1351 1352 assert(arena_run_tree_search(&bin->runs, mapelm) != NULL); 1353 1354 arena_run_tree_remove(&bin->runs, mapelm); 1355} 1356 1357static arena_run_t * 1358arena_bin_nonfull_run_tryget(arena_bin_t *bin) 1359{ 1360 arena_run_t *run = arena_bin_runs_first(bin); 1361 if (run != NULL) { 1362 arena_bin_runs_remove(bin, run); 1363 if (config_stats) 1364 bin->stats.reruns++; 1365 } 1366 return (run); 1367} 1368 1369static arena_run_t * 1370arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) 1371{ 1372 arena_run_t *run; 1373 size_t binind; 1374 arena_bin_info_t *bin_info; 1375 1376 /* Look for a usable run. */ 1377 run = arena_bin_nonfull_run_tryget(bin); 1378 if (run != NULL) 1379 return (run); 1380 /* No existing runs have any space available. */ 1381 1382 binind = arena_bin_index(arena, bin); 1383 bin_info = &arena_bin_info[binind]; 1384 1385 /* Allocate a new run. */ 1386 malloc_mutex_unlock(&bin->lock); 1387 /******************************/ 1388 malloc_mutex_lock(&arena->lock); 1389 run = arena_run_alloc_small(arena, bin_info->run_size, binind); 1390 if (run != NULL) { 1391 bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 1392 (uintptr_t)bin_info->bitmap_offset); 1393 1394 /* Initialize run internals. */ 1395 run->bin = bin; 1396 run->nextind = 0; 1397 run->nfree = bin_info->nregs; 1398 bitmap_init(bitmap, &bin_info->bitmap_info); 1399 } 1400 malloc_mutex_unlock(&arena->lock); 1401 /********************************/ 1402 malloc_mutex_lock(&bin->lock); 1403 if (run != NULL) { 1404 if (config_stats) { 1405 bin->stats.nruns++; 1406 bin->stats.curruns++; 1407 } 1408 return (run); 1409 } 1410 1411 /* 1412 * arena_run_alloc_small() failed, but another thread may have made 1413 * sufficient memory available while this one dropped bin->lock above, 1414 * so search one more time. 1415 */ 1416 run = arena_bin_nonfull_run_tryget(bin); 1417 if (run != NULL) 1418 return (run); 1419 1420 return (NULL); 1421} 1422 1423/* Re-fill bin->runcur, then call arena_run_reg_alloc(). */ 1424static void * 1425arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) 1426{ 1427 void *ret; 1428 size_t binind; 1429 arena_bin_info_t *bin_info; 1430 arena_run_t *run; 1431 1432 binind = arena_bin_index(arena, bin); 1433 bin_info = &arena_bin_info[binind]; 1434 bin->runcur = NULL; 1435 run = arena_bin_nonfull_run_get(arena, bin); 1436 if (bin->runcur != NULL && bin->runcur->nfree > 0) { 1437 /* 1438 * Another thread updated runcur while this one ran without the 1439 * bin lock in arena_bin_nonfull_run_get(). 1440 */ 1441 assert(bin->runcur->nfree > 0); 1442 ret = arena_run_reg_alloc(bin->runcur, bin_info); 1443 if (run != NULL) { 1444 arena_chunk_t *chunk; 1445 1446 /* 1447 * arena_run_alloc_small() may have allocated run, or 1448 * it may have pulled run from the bin's run tree. 1449 * Therefore it is unsafe to make any assumptions about 1450 * how run has previously been used, and 1451 * arena_bin_lower_run() must be called, as if a region 1452 * were just deallocated from the run. 1453 */ 1454 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1455 if (run->nfree == bin_info->nregs) 1456 arena_dalloc_bin_run(arena, chunk, run, bin); 1457 else 1458 arena_bin_lower_run(arena, chunk, run, bin); 1459 } 1460 return (ret); 1461 } 1462 1463 if (run == NULL) 1464 return (NULL); 1465 1466 bin->runcur = run; 1467 1468 assert(bin->runcur->nfree > 0); 1469 1470 return (arena_run_reg_alloc(bin->runcur, bin_info)); 1471} 1472 1473void 1474arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind, 1475 uint64_t prof_accumbytes) 1476{ 1477 unsigned i, nfill; 1478 arena_bin_t *bin; 1479 arena_run_t *run; 1480 void *ptr; 1481 1482 assert(tbin->ncached == 0); 1483 1484 if (config_prof && arena_prof_accum(arena, prof_accumbytes)) 1485 prof_idump(); 1486 bin = &arena->bins[binind]; 1487 malloc_mutex_lock(&bin->lock); 1488 for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> 1489 tbin->lg_fill_div); i < nfill; i++) { 1490 if ((run = bin->runcur) != NULL && run->nfree > 0) 1491 ptr = arena_run_reg_alloc(run, &arena_bin_info[binind]); 1492 else 1493 ptr = arena_bin_malloc_hard(arena, bin); 1494 if (ptr == NULL) 1495 break; 1496 if (config_fill && opt_junk) { 1497 arena_alloc_junk_small(ptr, &arena_bin_info[binind], 1498 true); 1499 } 1500 /* Insert such that low regions get used first. */ 1501 tbin->avail[nfill - 1 - i] = ptr; 1502 } 1503 if (config_stats) { 1504 bin->stats.allocated += i * arena_bin_info[binind].reg_size; 1505 bin->stats.nmalloc += i; 1506 bin->stats.nrequests += tbin->tstats.nrequests; 1507 bin->stats.nfills++; 1508 tbin->tstats.nrequests = 0; 1509 } 1510 malloc_mutex_unlock(&bin->lock); 1511 tbin->ncached = i; 1512} 1513 1514void 1515arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, bool zero) 1516{ 1517 1518 if (zero) { 1519 size_t redzone_size = bin_info->redzone_size; 1520 memset((void *)((uintptr_t)ptr - redzone_size), 0xa5, 1521 redzone_size); 1522 memset((void *)((uintptr_t)ptr + bin_info->reg_size), 0xa5, 1523 redzone_size); 1524 } else { 1525 memset((void *)((uintptr_t)ptr - bin_info->redzone_size), 0xa5, 1526 bin_info->reg_interval); 1527 } 1528} 1529 1530#ifdef JEMALLOC_JET 1531#undef arena_redzone_corruption 1532#define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption_impl) 1533#endif 1534static void 1535arena_redzone_corruption(void *ptr, size_t usize, bool after, 1536 size_t offset, uint8_t byte) 1537{ 1538 1539 malloc_printf("<jemalloc>: Corrupt redzone %zu byte%s %s %p " 1540 "(size %zu), byte=%#x\n", offset, (offset == 1) ? "" : "s", 1541 after ? "after" : "before", ptr, usize, byte); 1542} 1543#ifdef JEMALLOC_JET 1544#undef arena_redzone_corruption 1545#define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption) 1546arena_redzone_corruption_t *arena_redzone_corruption = 1547 JEMALLOC_N(arena_redzone_corruption_impl); 1548#endif 1549 1550static void 1551arena_redzones_validate(void *ptr, arena_bin_info_t *bin_info, bool reset) 1552{ 1553 size_t size = bin_info->reg_size; 1554 size_t redzone_size = bin_info->redzone_size; 1555 size_t i; 1556 bool error = false; 1557 1558 for (i = 1; i <= redzone_size; i++) { 1559 uint8_t *byte = (uint8_t *)((uintptr_t)ptr - i); 1560 if (*byte != 0xa5) { 1561 error = true; 1562 arena_redzone_corruption(ptr, size, false, i, *byte); 1563 if (reset) 1564 *byte = 0xa5; 1565 } 1566 } 1567 for (i = 0; i < redzone_size; i++) { 1568 uint8_t *byte = (uint8_t *)((uintptr_t)ptr + size + i); 1569 if (*byte != 0xa5) { 1570 error = true; 1571 arena_redzone_corruption(ptr, size, true, i, *byte); 1572 if (reset) 1573 *byte = 0xa5; 1574 } 1575 } 1576 if (opt_abort && error) 1577 abort(); 1578} 1579 1580#ifdef JEMALLOC_JET 1581#undef arena_dalloc_junk_small 1582#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small_impl) 1583#endif 1584void 1585arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info) 1586{ 1587 size_t redzone_size = bin_info->redzone_size; 1588 1589 arena_redzones_validate(ptr, bin_info, false); 1590 memset((void *)((uintptr_t)ptr - redzone_size), 0x5a, 1591 bin_info->reg_interval); 1592} 1593#ifdef JEMALLOC_JET 1594#undef arena_dalloc_junk_small 1595#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small) 1596arena_dalloc_junk_small_t *arena_dalloc_junk_small = 1597 JEMALLOC_N(arena_dalloc_junk_small_impl); 1598#endif 1599 1600void 1601arena_quarantine_junk_small(void *ptr, size_t usize) 1602{ 1603 size_t binind; 1604 arena_bin_info_t *bin_info; 1605 cassert(config_fill); 1606 assert(opt_junk); 1607 assert(opt_quarantine); 1608 assert(usize <= SMALL_MAXCLASS); 1609 1610 binind = SMALL_SIZE2BIN(usize); 1611 bin_info = &arena_bin_info[binind]; 1612 arena_redzones_validate(ptr, bin_info, true); 1613} 1614 1615void * 1616arena_malloc_small(arena_t *arena, size_t size, bool zero) 1617{ 1618 void *ret; 1619 arena_bin_t *bin; 1620 arena_run_t *run; 1621 size_t binind; 1622 1623 binind = SMALL_SIZE2BIN(size); 1624 assert(binind < NBINS); 1625 bin = &arena->bins[binind]; 1626 size = small_bin2size[binind]; 1627 1628 malloc_mutex_lock(&bin->lock); 1629 if ((run = bin->runcur) != NULL && run->nfree > 0) 1630 ret = arena_run_reg_alloc(run, &arena_bin_info[binind]); 1631 else 1632 ret = arena_bin_malloc_hard(arena, bin); 1633 1634 if (ret == NULL) { 1635 malloc_mutex_unlock(&bin->lock); 1636 return (NULL); 1637 } 1638 1639 if (config_stats) { 1640 bin->stats.allocated += size; 1641 bin->stats.nmalloc++; 1642 bin->stats.nrequests++; 1643 } 1644 malloc_mutex_unlock(&bin->lock); 1645 if (config_prof && isthreaded == false && arena_prof_accum(arena, size)) 1646 prof_idump(); 1647 1648 if (zero == false) { 1649 if (config_fill) { 1650 if (opt_junk) { 1651 arena_alloc_junk_small(ret, 1652 &arena_bin_info[binind], false); 1653 } else if (opt_zero) 1654 memset(ret, 0, size); 1655 } 1656 VALGRIND_MAKE_MEM_UNDEFINED(ret, size); 1657 } else { 1658 if (config_fill && opt_junk) { 1659 arena_alloc_junk_small(ret, &arena_bin_info[binind], 1660 true); 1661 } 1662 VALGRIND_MAKE_MEM_UNDEFINED(ret, size); 1663 memset(ret, 0, size); 1664 } 1665 1666 return (ret); 1667} 1668 1669void * 1670arena_malloc_large(arena_t *arena, size_t size, bool zero) 1671{ 1672 void *ret; 1673 UNUSED bool idump; 1674 1675 /* Large allocation. */ 1676 size = PAGE_CEILING(size); 1677 malloc_mutex_lock(&arena->lock); 1678 ret = (void *)arena_run_alloc_large(arena, size, zero); 1679 if (ret == NULL) { 1680 malloc_mutex_unlock(&arena->lock); 1681 return (NULL); 1682 } 1683 if (config_stats) { 1684 arena->stats.nmalloc_large++; 1685 arena->stats.nrequests_large++; 1686 arena->stats.allocated_large += size; 1687 arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 1688 arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 1689 arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 1690 } 1691 if (config_prof) 1692 idump = arena_prof_accum_locked(arena, size); 1693 malloc_mutex_unlock(&arena->lock); 1694 if (config_prof && idump) 1695 prof_idump(); 1696 1697 if (zero == false) { 1698 if (config_fill) { 1699 if (opt_junk) 1700 memset(ret, 0xa5, size); 1701 else if (opt_zero) 1702 memset(ret, 0, size); 1703 } 1704 } 1705 1706 return (ret); 1707} 1708 1709/* Only handles large allocations that require more than page alignment. */ 1710void * 1711arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero) 1712{ 1713 void *ret; 1714 size_t alloc_size, leadsize, trailsize; 1715 arena_run_t *run; 1716 arena_chunk_t *chunk; 1717 1718 assert((size & PAGE_MASK) == 0); 1719 1720 alignment = PAGE_CEILING(alignment); 1721 alloc_size = size + alignment - PAGE; 1722 1723 malloc_mutex_lock(&arena->lock); 1724 run = arena_run_alloc_large(arena, alloc_size, false); 1725 if (run == NULL) { 1726 malloc_mutex_unlock(&arena->lock); 1727 return (NULL); 1728 } 1729 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1730 1731 leadsize = ALIGNMENT_CEILING((uintptr_t)run, alignment) - 1732 (uintptr_t)run; 1733 assert(alloc_size >= leadsize + size); 1734 trailsize = alloc_size - leadsize - size; 1735 ret = (void *)((uintptr_t)run + leadsize); 1736 if (leadsize != 0) { 1737 arena_run_trim_head(arena, chunk, run, alloc_size, alloc_size - 1738 leadsize); 1739 } 1740 if (trailsize != 0) { 1741 arena_run_trim_tail(arena, chunk, ret, size + trailsize, size, 1742 false); 1743 } 1744 arena_run_init_large(arena, (arena_run_t *)ret, size, zero); 1745 1746 if (config_stats) { 1747 arena->stats.nmalloc_large++; 1748 arena->stats.nrequests_large++; 1749 arena->stats.allocated_large += size; 1750 arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 1751 arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 1752 arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 1753 } 1754 malloc_mutex_unlock(&arena->lock); 1755 1756 if (config_fill && zero == false) { 1757 if (opt_junk) 1758 memset(ret, 0xa5, size); 1759 else if (opt_zero) 1760 memset(ret, 0, size); 1761 } 1762 return (ret); 1763} 1764 1765void 1766arena_prof_promoted(const void *ptr, size_t size) 1767{ 1768 arena_chunk_t *chunk; 1769 size_t pageind, binind; 1770 1771 cassert(config_prof); 1772 assert(ptr != NULL); 1773 assert(CHUNK_ADDR2BASE(ptr) != ptr); 1774 assert(isalloc(ptr, false) == PAGE); 1775 assert(isalloc(ptr, true) == PAGE); 1776 assert(size <= SMALL_MAXCLASS); 1777 1778 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1779 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1780 binind = SMALL_SIZE2BIN(size); 1781 assert(binind < NBINS); 1782 arena_mapbits_large_binind_set(chunk, pageind, binind); 1783 1784 assert(isalloc(ptr, false) == PAGE); 1785 assert(isalloc(ptr, true) == size); 1786} 1787 1788static void 1789arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, 1790 arena_bin_t *bin) 1791{ 1792 1793 /* Dissociate run from bin. */ 1794 if (run == bin->runcur) 1795 bin->runcur = NULL; 1796 else { 1797 size_t binind = arena_bin_index(chunk->arena, bin); 1798 arena_bin_info_t *bin_info = &arena_bin_info[binind]; 1799 1800 if (bin_info->nregs != 1) { 1801 /* 1802 * This block's conditional is necessary because if the 1803 * run only contains one region, then it never gets 1804 * inserted into the non-full runs tree. 1805 */ 1806 arena_bin_runs_remove(bin, run); 1807 } 1808 } 1809} 1810 1811static void 1812arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1813 arena_bin_t *bin) 1814{ 1815 size_t binind; 1816 arena_bin_info_t *bin_info; 1817 size_t npages, run_ind, past; 1818 1819 assert(run != bin->runcur); 1820 assert(arena_run_tree_search(&bin->runs, 1821 arena_mapp_get(chunk, ((uintptr_t)run-(uintptr_t)chunk)>>LG_PAGE)) 1822 == NULL); 1823 1824 binind = arena_bin_index(chunk->arena, run->bin); 1825 bin_info = &arena_bin_info[binind]; 1826 1827 malloc_mutex_unlock(&bin->lock); 1828 /******************************/ 1829 npages = bin_info->run_size >> LG_PAGE; 1830 run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 1831 past = (size_t)(PAGE_CEILING((uintptr_t)run + 1832 (uintptr_t)bin_info->reg0_offset + (uintptr_t)(run->nextind * 1833 bin_info->reg_interval - bin_info->redzone_size) - 1834 (uintptr_t)chunk) >> LG_PAGE); 1835 malloc_mutex_lock(&arena->lock); 1836 1837 /* 1838 * If the run was originally clean, and some pages were never touched, 1839 * trim the clean pages before deallocating the dirty portion of the 1840 * run. 1841 */ 1842 assert(arena_mapbits_dirty_get(chunk, run_ind) == 1843 arena_mapbits_dirty_get(chunk, run_ind+npages-1)); 1844 if (arena_mapbits_dirty_get(chunk, run_ind) == 0 && past - run_ind < 1845 npages) { 1846 /* Trim clean pages. Convert to large run beforehand. */ 1847 assert(npages > 0); 1848 arena_mapbits_large_set(chunk, run_ind, bin_info->run_size, 0); 1849 arena_mapbits_large_set(chunk, run_ind+npages-1, 0, 0); 1850 arena_run_trim_tail(arena, chunk, run, (npages << LG_PAGE), 1851 ((past - run_ind) << LG_PAGE), false); 1852 /* npages = past - run_ind; */ 1853 } 1854 arena_run_dalloc(arena, run, true, false); 1855 malloc_mutex_unlock(&arena->lock); 1856 /****************************/ 1857 malloc_mutex_lock(&bin->lock); 1858 if (config_stats) 1859 bin->stats.curruns--; 1860} 1861 1862static void 1863arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1864 arena_bin_t *bin) 1865{ 1866 1867 /* 1868 * Make sure that if bin->runcur is non-NULL, it refers to the lowest 1869 * non-full run. It is okay to NULL runcur out rather than proactively 1870 * keeping it pointing at the lowest non-full run. 1871 */ 1872 if ((uintptr_t)run < (uintptr_t)bin->runcur) { 1873 /* Switch runcur. */ 1874 if (bin->runcur->nfree > 0) 1875 arena_bin_runs_insert(bin, bin->runcur); 1876 bin->runcur = run; 1877 if (config_stats) 1878 bin->stats.reruns++; 1879 } else 1880 arena_bin_runs_insert(bin, run); 1881} 1882 1883void 1884arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1885 arena_chunk_map_t *mapelm) 1886{ 1887 size_t pageind; 1888 arena_run_t *run; 1889 arena_bin_t *bin; 1890 arena_bin_info_t *bin_info; 1891 size_t size, binind; 1892 1893 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1894 run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 1895 arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); 1896 bin = run->bin; 1897 binind = arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, pageind)); 1898 bin_info = &arena_bin_info[binind]; 1899 if (config_fill || config_stats) 1900 size = bin_info->reg_size; 1901 1902 if (config_fill && opt_junk) 1903 arena_dalloc_junk_small(ptr, bin_info); 1904 1905 arena_run_reg_dalloc(run, ptr); 1906 if (run->nfree == bin_info->nregs) { 1907 arena_dissociate_bin_run(chunk, run, bin); 1908 arena_dalloc_bin_run(arena, chunk, run, bin); 1909 } else if (run->nfree == 1 && run != bin->runcur) 1910 arena_bin_lower_run(arena, chunk, run, bin); 1911 1912 if (config_stats) { 1913 bin->stats.allocated -= size; 1914 bin->stats.ndalloc++; 1915 } 1916} 1917 1918void 1919arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1920 size_t pageind, arena_chunk_map_t *mapelm) 1921{ 1922 arena_run_t *run; 1923 arena_bin_t *bin; 1924 1925 run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 1926 arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); 1927 bin = run->bin; 1928 malloc_mutex_lock(&bin->lock); 1929 arena_dalloc_bin_locked(arena, chunk, ptr, mapelm); 1930 malloc_mutex_unlock(&bin->lock); 1931} 1932 1933void 1934arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1935 size_t pageind) 1936{ 1937 arena_chunk_map_t *mapelm; 1938 1939 if (config_debug) { 1940 /* arena_ptr_small_binind_get() does extra sanity checking. */ 1941 assert(arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, 1942 pageind)) != BININD_INVALID); 1943 } 1944 mapelm = arena_mapp_get(chunk, pageind); 1945 arena_dalloc_bin(arena, chunk, ptr, pageind, mapelm); 1946} 1947 1948#ifdef JEMALLOC_JET 1949#undef arena_dalloc_junk_large 1950#define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large_impl) 1951#endif 1952static void 1953arena_dalloc_junk_large(void *ptr, size_t usize) 1954{ 1955 1956 if (config_fill && opt_junk) 1957 memset(ptr, 0x5a, usize); 1958} 1959#ifdef JEMALLOC_JET 1960#undef arena_dalloc_junk_large 1961#define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large) 1962arena_dalloc_junk_large_t *arena_dalloc_junk_large = 1963 JEMALLOC_N(arena_dalloc_junk_large_impl); 1964#endif 1965 1966void 1967arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr) 1968{ 1969 1970 if (config_fill || config_stats) { 1971 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1972 size_t usize = arena_mapbits_large_size_get(chunk, pageind); 1973 1974 arena_dalloc_junk_large(ptr, usize); 1975 if (config_stats) { 1976 arena->stats.ndalloc_large++; 1977 arena->stats.allocated_large -= usize; 1978 arena->stats.lstats[(usize >> LG_PAGE) - 1].ndalloc++; 1979 arena->stats.lstats[(usize >> LG_PAGE) - 1].curruns--; 1980 } 1981 } 1982 1983 arena_run_dalloc(arena, (arena_run_t *)ptr, true, false); 1984} 1985 1986void 1987arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr) 1988{ 1989 1990 malloc_mutex_lock(&arena->lock); 1991 arena_dalloc_large_locked(arena, chunk, ptr); 1992 malloc_mutex_unlock(&arena->lock); 1993} 1994 1995static void 1996arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1997 size_t oldsize, size_t size) 1998{ 1999 2000 assert(size < oldsize); 2001 2002 /* 2003 * Shrink the run, and make trailing pages available for other 2004 * allocations. 2005 */ 2006 malloc_mutex_lock(&arena->lock); 2007 arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size, 2008 true); 2009 if (config_stats) { 2010 arena->stats.ndalloc_large++; 2011 arena->stats.allocated_large -= oldsize; 2012 arena->stats.lstats[(oldsize >> LG_PAGE) - 1].ndalloc++; 2013 arena->stats.lstats[(oldsize >> LG_PAGE) - 1].curruns--; 2014 2015 arena->stats.nmalloc_large++; 2016 arena->stats.nrequests_large++; 2017 arena->stats.allocated_large += size; 2018 arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 2019 arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 2020 arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 2021 } 2022 malloc_mutex_unlock(&arena->lock); 2023} 2024 2025static bool 2026arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, 2027 size_t oldsize, size_t size, size_t extra, bool zero) 2028{ 2029 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 2030 size_t npages = oldsize >> LG_PAGE; 2031 size_t followsize; 2032 2033 assert(oldsize == arena_mapbits_large_size_get(chunk, pageind)); 2034 2035 /* Try to extend the run. */ 2036 assert(size + extra > oldsize); 2037 malloc_mutex_lock(&arena->lock); 2038 if (pageind + npages < chunk_npages && 2039 arena_mapbits_allocated_get(chunk, pageind+npages) == 0 && 2040 (followsize = arena_mapbits_unallocated_size_get(chunk, 2041 pageind+npages)) >= size - oldsize) { 2042 /* 2043 * The next run is available and sufficiently large. Split the 2044 * following run, then merge the first part with the existing 2045 * allocation. 2046 */ 2047 size_t flag_dirty; 2048 size_t splitsize = (oldsize + followsize <= size + extra) 2049 ? followsize : size + extra - oldsize; 2050 arena_run_split_large(arena, (arena_run_t *)((uintptr_t)chunk + 2051 ((pageind+npages) << LG_PAGE)), splitsize, zero); 2052 2053 size = oldsize + splitsize; 2054 npages = size >> LG_PAGE; 2055 2056 /* 2057 * Mark the extended run as dirty if either portion of the run 2058 * was dirty before allocation. This is rather pedantic, 2059 * because there's not actually any sequence of events that 2060 * could cause the resulting run to be passed to 2061 * arena_run_dalloc() with the dirty argument set to false 2062 * (which is when dirty flag consistency would really matter). 2063 */ 2064 flag_dirty = arena_mapbits_dirty_get(chunk, pageind) | 2065 arena_mapbits_dirty_get(chunk, pageind+npages-1); 2066 arena_mapbits_large_set(chunk, pageind, size, flag_dirty); 2067 arena_mapbits_large_set(chunk, pageind+npages-1, 0, flag_dirty); 2068 2069 if (config_stats) { 2070 arena->stats.ndalloc_large++; 2071 arena->stats.allocated_large -= oldsize; 2072 arena->stats.lstats[(oldsize >> LG_PAGE) - 1].ndalloc++; 2073 arena->stats.lstats[(oldsize >> LG_PAGE) - 1].curruns--; 2074 2075 arena->stats.nmalloc_large++; 2076 arena->stats.nrequests_large++; 2077 arena->stats.allocated_large += size; 2078 arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 2079 arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 2080 arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 2081 } 2082 malloc_mutex_unlock(&arena->lock); 2083 return (false); 2084 } 2085 malloc_mutex_unlock(&arena->lock); 2086 2087 return (true); 2088} 2089 2090#ifdef JEMALLOC_JET 2091#undef arena_ralloc_junk_large 2092#define arena_ralloc_junk_large JEMALLOC_N(arena_ralloc_junk_large_impl) 2093#endif 2094static void 2095arena_ralloc_junk_large(void *ptr, size_t old_usize, size_t usize) 2096{ 2097 2098 if (config_fill && opt_junk) { 2099 memset((void *)((uintptr_t)ptr + usize), 0x5a, 2100 old_usize - usize); 2101 } 2102} 2103#ifdef JEMALLOC_JET 2104#undef arena_ralloc_junk_large 2105#define arena_ralloc_junk_large JEMALLOC_N(arena_ralloc_junk_large) 2106arena_ralloc_junk_large_t *arena_ralloc_junk_large = 2107 JEMALLOC_N(arena_ralloc_junk_large_impl); 2108#endif 2109 2110/* 2111 * Try to resize a large allocation, in order to avoid copying. This will 2112 * always fail if growing an object, and the following run is already in use. 2113 */ 2114static bool 2115arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, 2116 bool zero) 2117{ 2118 size_t psize; 2119 2120 psize = PAGE_CEILING(size + extra); 2121 if (psize == oldsize) { 2122 /* Same size class. */ 2123 return (false); 2124 } else { 2125 arena_chunk_t *chunk; 2126 arena_t *arena; 2127 2128 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 2129 arena = chunk->arena; 2130 2131 if (psize < oldsize) { 2132 /* Fill before shrinking in order avoid a race. */ 2133 arena_ralloc_junk_large(ptr, oldsize, psize); 2134 arena_ralloc_large_shrink(arena, chunk, ptr, oldsize, 2135 psize); 2136 return (false); 2137 } else { 2138 bool ret = arena_ralloc_large_grow(arena, chunk, ptr, 2139 oldsize, PAGE_CEILING(size), 2140 psize - PAGE_CEILING(size), zero); 2141 if (config_fill && ret == false && zero == false) { 2142 if (opt_junk) { 2143 memset((void *)((uintptr_t)ptr + 2144 oldsize), 0xa5, isalloc(ptr, 2145 config_prof) - oldsize); 2146 } else if (opt_zero) { 2147 memset((void *)((uintptr_t)ptr + 2148 oldsize), 0, isalloc(ptr, 2149 config_prof) - oldsize); 2150 } 2151 } 2152 return (ret); 2153 } 2154 } 2155} 2156 2157bool 2158arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, 2159 bool zero) 2160{ 2161 2162 /* 2163 * Avoid moving the allocation if the size class can be left the same. 2164 */ 2165 if (oldsize <= arena_maxclass) { 2166 if (oldsize <= SMALL_MAXCLASS) { 2167 assert(arena_bin_info[SMALL_SIZE2BIN(oldsize)].reg_size 2168 == oldsize); 2169 if ((size + extra <= SMALL_MAXCLASS && 2170 SMALL_SIZE2BIN(size + extra) == 2171 SMALL_SIZE2BIN(oldsize)) || (size <= oldsize && 2172 size + extra >= oldsize)) 2173 return (false); 2174 } else { 2175 assert(size <= arena_maxclass); 2176 if (size + extra > SMALL_MAXCLASS) { 2177 if (arena_ralloc_large(ptr, oldsize, size, 2178 extra, zero) == false) 2179 return (false); 2180 } 2181 } 2182 } 2183 2184 /* Reallocation would require a move. */ 2185 return (true); 2186} 2187 2188void * 2189arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, 2190 size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, 2191 bool try_tcache_dalloc) 2192{ 2193 void *ret; 2194 size_t copysize; 2195 2196 /* Try to avoid moving the allocation. */ 2197 if (arena_ralloc_no_move(ptr, oldsize, size, extra, zero) == false) 2198 return (ptr); 2199 2200 /* 2201 * size and oldsize are different enough that we need to move the 2202 * object. In that case, fall back to allocating new space and 2203 * copying. 2204 */ 2205 if (alignment != 0) { 2206 size_t usize = sa2u(size + extra, alignment); 2207 if (usize == 0) 2208 return (NULL); 2209 ret = ipalloct(usize, alignment, zero, try_tcache_alloc, arena); 2210 } else 2211 ret = arena_malloc(arena, size + extra, zero, try_tcache_alloc); 2212 2213 if (ret == NULL) { 2214 if (extra == 0) 2215 return (NULL); 2216 /* Try again, this time without extra. */ 2217 if (alignment != 0) { 2218 size_t usize = sa2u(size, alignment); 2219 if (usize == 0) 2220 return (NULL); 2221 ret = ipalloct(usize, alignment, zero, try_tcache_alloc, 2222 arena); 2223 } else 2224 ret = arena_malloc(arena, size, zero, try_tcache_alloc); 2225 2226 if (ret == NULL) 2227 return (NULL); 2228 } 2229 2230 /* Junk/zero-filling were already done by ipalloc()/arena_malloc(). */ 2231 2232 /* 2233 * Copy at most size bytes (not size+extra), since the caller has no 2234 * expectation that the extra bytes will be reliably preserved. 2235 */ 2236 copysize = (size < oldsize) ? size : oldsize; 2237 VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize); 2238 memcpy(ret, ptr, copysize); 2239 iqalloct(ptr, try_tcache_dalloc); 2240 return (ret); 2241} 2242 2243dss_prec_t 2244arena_dss_prec_get(arena_t *arena) 2245{ 2246 dss_prec_t ret; 2247 2248 malloc_mutex_lock(&arena->lock); 2249 ret = arena->dss_prec; 2250 malloc_mutex_unlock(&arena->lock); 2251 return (ret); 2252} 2253 2254void 2255arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) 2256{ 2257 2258 malloc_mutex_lock(&arena->lock); 2259 arena->dss_prec = dss_prec; 2260 malloc_mutex_unlock(&arena->lock); 2261} 2262 2263void 2264arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, 2265 size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, 2266 malloc_large_stats_t *lstats) 2267{ 2268 unsigned i; 2269 2270 malloc_mutex_lock(&arena->lock); 2271 *dss = dss_prec_names[arena->dss_prec]; 2272 *nactive += arena->nactive; 2273 *ndirty += arena->ndirty; 2274 2275 astats->mapped += arena->stats.mapped; 2276 astats->npurge += arena->stats.npurge; 2277 astats->nmadvise += arena->stats.nmadvise; 2278 astats->purged += arena->stats.purged; 2279 astats->allocated_large += arena->stats.allocated_large; 2280 astats->nmalloc_large += arena->stats.nmalloc_large; 2281 astats->ndalloc_large += arena->stats.ndalloc_large; 2282 astats->nrequests_large += arena->stats.nrequests_large; 2283 2284 for (i = 0; i < nlclasses; i++) { 2285 lstats[i].nmalloc += arena->stats.lstats[i].nmalloc; 2286 lstats[i].ndalloc += arena->stats.lstats[i].ndalloc; 2287 lstats[i].nrequests += arena->stats.lstats[i].nrequests; 2288 lstats[i].curruns += arena->stats.lstats[i].curruns; 2289 } 2290 malloc_mutex_unlock(&arena->lock); 2291 2292 for (i = 0; i < NBINS; i++) { 2293 arena_bin_t *bin = &arena->bins[i]; 2294 2295 malloc_mutex_lock(&bin->lock); 2296 bstats[i].allocated += bin->stats.allocated; 2297 bstats[i].nmalloc += bin->stats.nmalloc; 2298 bstats[i].ndalloc += bin->stats.ndalloc; 2299 bstats[i].nrequests += bin->stats.nrequests; 2300 if (config_tcache) { 2301 bstats[i].nfills += bin->stats.nfills; 2302 bstats[i].nflushes += bin->stats.nflushes; 2303 } 2304 bstats[i].nruns += bin->stats.nruns; 2305 bstats[i].reruns += bin->stats.reruns; 2306 bstats[i].curruns += bin->stats.curruns; 2307 malloc_mutex_unlock(&bin->lock); 2308 } 2309} 2310 2311bool 2312arena_new(arena_t *arena, unsigned ind) 2313{ 2314 unsigned i; 2315 arena_bin_t *bin; 2316 2317 arena->ind = ind; 2318 arena->nthreads = 0; 2319 2320 if (malloc_mutex_init(&arena->lock)) 2321 return (true); 2322 2323 if (config_stats) { 2324 memset(&arena->stats, 0, sizeof(arena_stats_t)); 2325 arena->stats.lstats = 2326 (malloc_large_stats_t *)base_alloc(nlclasses * 2327 sizeof(malloc_large_stats_t)); 2328 if (arena->stats.lstats == NULL) 2329 return (true); 2330 memset(arena->stats.lstats, 0, nlclasses * 2331 sizeof(malloc_large_stats_t)); 2332 if (config_tcache) 2333 ql_new(&arena->tcache_ql); 2334 } 2335 2336 if (config_prof) 2337 arena->prof_accumbytes = 0; 2338 2339 arena->dss_prec = chunk_dss_prec_get(); 2340 2341 /* Initialize chunks. */ 2342 arena_chunk_dirty_new(&arena->chunks_dirty); 2343 arena->spare = NULL; 2344 2345 arena->nactive = 0; 2346 arena->ndirty = 0; 2347 arena->npurgatory = 0; 2348 2349 arena_avail_tree_new(&arena->runs_avail); 2350 2351 /* Initialize bins. */ 2352 for (i = 0; i < NBINS; i++) { 2353 bin = &arena->bins[i]; 2354 if (malloc_mutex_init(&bin->lock)) 2355 return (true); 2356 bin->runcur = NULL; 2357 arena_run_tree_new(&bin->runs); 2358 if (config_stats) 2359 memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 2360 } 2361 2362 return (false); 2363} 2364 2365/* 2366 * Calculate bin_info->run_size such that it meets the following constraints: 2367 * 2368 * *) bin_info->run_size >= min_run_size 2369 * *) bin_info->run_size <= arena_maxclass 2370 * *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed). 2371 * *) bin_info->nregs <= RUN_MAXREGS 2372 * 2373 * bin_info->nregs, bin_info->bitmap_offset, and bin_info->reg0_offset are also 2374 * calculated here, since these settings are all interdependent. 2375 */ 2376static size_t 2377bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) 2378{ 2379 size_t pad_size; 2380 size_t try_run_size, good_run_size; 2381 uint32_t try_nregs, good_nregs; 2382 uint32_t try_hdr_size, good_hdr_size; 2383 uint32_t try_bitmap_offset, good_bitmap_offset; 2384 uint32_t try_redzone0_offset, good_redzone0_offset; 2385 2386 assert(min_run_size >= PAGE); 2387 assert(min_run_size <= arena_maxclass); 2388 2389 /* 2390 * Determine redzone size based on minimum alignment and minimum 2391 * redzone size. Add padding to the end of the run if it is needed to 2392 * align the regions. The padding allows each redzone to be half the 2393 * minimum alignment; without the padding, each redzone would have to 2394 * be twice as large in order to maintain alignment. 2395 */ 2396 if (config_fill && opt_redzone) { 2397 size_t align_min = ZU(1) << (ffs(bin_info->reg_size) - 1); 2398 if (align_min <= REDZONE_MINSIZE) { 2399 bin_info->redzone_size = REDZONE_MINSIZE; 2400 pad_size = 0; 2401 } else { 2402 bin_info->redzone_size = align_min >> 1; 2403 pad_size = bin_info->redzone_size; 2404 } 2405 } else { 2406 bin_info->redzone_size = 0; 2407 pad_size = 0; 2408 } 2409 bin_info->reg_interval = bin_info->reg_size + 2410 (bin_info->redzone_size << 1); 2411 2412 /* 2413 * Calculate known-valid settings before entering the run_size 2414 * expansion loop, so that the first part of the loop always copies 2415 * valid settings. 2416 * 2417 * The do..while loop iteratively reduces the number of regions until 2418 * the run header and the regions no longer overlap. A closed formula 2419 * would be quite messy, since there is an interdependency between the 2420 * header's mask length and the number of regions. 2421 */ 2422 try_run_size = min_run_size; 2423 try_nregs = ((try_run_size - sizeof(arena_run_t)) / 2424 bin_info->reg_interval) 2425 + 1; /* Counter-act try_nregs-- in loop. */ 2426 if (try_nregs > RUN_MAXREGS) { 2427 try_nregs = RUN_MAXREGS 2428 + 1; /* Counter-act try_nregs-- in loop. */ 2429 } 2430 do { 2431 try_nregs--; 2432 try_hdr_size = sizeof(arena_run_t); 2433 /* Pad to a long boundary. */ 2434 try_hdr_size = LONG_CEILING(try_hdr_size); 2435 try_bitmap_offset = try_hdr_size; 2436 /* Add space for bitmap. */ 2437 try_hdr_size += bitmap_size(try_nregs); 2438 try_redzone0_offset = try_run_size - (try_nregs * 2439 bin_info->reg_interval) - pad_size; 2440 } while (try_hdr_size > try_redzone0_offset); 2441 2442 /* run_size expansion loop. */ 2443 do { 2444 /* 2445 * Copy valid settings before trying more aggressive settings. 2446 */ 2447 good_run_size = try_run_size; 2448 good_nregs = try_nregs; 2449 good_hdr_size = try_hdr_size; 2450 good_bitmap_offset = try_bitmap_offset; 2451 good_redzone0_offset = try_redzone0_offset; 2452 2453 /* Try more aggressive settings. */ 2454 try_run_size += PAGE; 2455 try_nregs = ((try_run_size - sizeof(arena_run_t) - pad_size) / 2456 bin_info->reg_interval) 2457 + 1; /* Counter-act try_nregs-- in loop. */ 2458 if (try_nregs > RUN_MAXREGS) { 2459 try_nregs = RUN_MAXREGS 2460 + 1; /* Counter-act try_nregs-- in loop. */ 2461 } 2462 do { 2463 try_nregs--; 2464 try_hdr_size = sizeof(arena_run_t); 2465 /* Pad to a long boundary. */ 2466 try_hdr_size = LONG_CEILING(try_hdr_size); 2467 try_bitmap_offset = try_hdr_size; 2468 /* Add space for bitmap. */ 2469 try_hdr_size += bitmap_size(try_nregs); 2470 try_redzone0_offset = try_run_size - (try_nregs * 2471 bin_info->reg_interval) - pad_size; 2472 } while (try_hdr_size > try_redzone0_offset); 2473 } while (try_run_size <= arena_maxclass 2474 && RUN_MAX_OVRHD * (bin_info->reg_interval << 3) > 2475 RUN_MAX_OVRHD_RELAX 2476 && (try_redzone0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size 2477 && try_nregs < RUN_MAXREGS); 2478 2479 assert(good_hdr_size <= good_redzone0_offset); 2480 2481 /* Copy final settings. */ 2482 bin_info->run_size = good_run_size; 2483 bin_info->nregs = good_nregs; 2484 bin_info->bitmap_offset = good_bitmap_offset; 2485 bin_info->reg0_offset = good_redzone0_offset + bin_info->redzone_size; 2486 2487 assert(bin_info->reg0_offset - bin_info->redzone_size + (bin_info->nregs 2488 * bin_info->reg_interval) + pad_size == bin_info->run_size); 2489 2490 return (good_run_size); 2491} 2492 2493static void 2494bin_info_init(void) 2495{ 2496 arena_bin_info_t *bin_info; 2497 size_t prev_run_size = PAGE; 2498 2499#define SIZE_CLASS(bin, delta, size) \ 2500 bin_info = &arena_bin_info[bin]; \ 2501 bin_info->reg_size = size; \ 2502 prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size);\ 2503 bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); 2504 SIZE_CLASSES 2505#undef SIZE_CLASS 2506} 2507 2508void 2509arena_boot(void) 2510{ 2511 size_t header_size; 2512 unsigned i; 2513 2514 /* 2515 * Compute the header size such that it is large enough to contain the 2516 * page map. The page map is biased to omit entries for the header 2517 * itself, so some iteration is necessary to compute the map bias. 2518 * 2519 * 1) Compute safe header_size and map_bias values that include enough 2520 * space for an unbiased page map. 2521 * 2) Refine map_bias based on (1) to omit the header pages in the page 2522 * map. The resulting map_bias may be one too small. 2523 * 3) Refine map_bias based on (2). The result will be >= the result 2524 * from (2), and will always be correct. 2525 */ 2526 map_bias = 0; 2527 for (i = 0; i < 3; i++) { 2528 header_size = offsetof(arena_chunk_t, map) + 2529 (sizeof(arena_chunk_map_t) * (chunk_npages-map_bias)); 2530 map_bias = (header_size >> LG_PAGE) + ((header_size & PAGE_MASK) 2531 != 0); 2532 } 2533 assert(map_bias > 0); 2534 2535 arena_maxclass = chunksize - (map_bias << LG_PAGE); 2536 2537 bin_info_init(); 2538} 2539 2540void 2541arena_prefork(arena_t *arena) 2542{ 2543 unsigned i; 2544 2545 malloc_mutex_prefork(&arena->lock); 2546 for (i = 0; i < NBINS; i++) 2547 malloc_mutex_prefork(&arena->bins[i].lock); 2548} 2549 2550void 2551arena_postfork_parent(arena_t *arena) 2552{ 2553 unsigned i; 2554 2555 for (i = 0; i < NBINS; i++) 2556 malloc_mutex_postfork_parent(&arena->bins[i].lock); 2557 malloc_mutex_postfork_parent(&arena->lock); 2558} 2559 2560void 2561arena_postfork_child(arena_t *arena) 2562{ 2563 unsigned i; 2564 2565 for (i = 0; i < NBINS; i++) 2566 malloc_mutex_postfork_child(&arena->bins[i].lock); 2567 malloc_mutex_postfork_child(&arena->lock); 2568} 2569