arena.c revision ee41ad409a43d12900a5a3108f6c14f84e4eb0eb
1#define JEMALLOC_ARENA_C_ 2#include "jemalloc/internal/jemalloc_internal.h" 3 4/******************************************************************************/ 5/* Data. */ 6 7ssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; 8arena_bin_info_t arena_bin_info[NBINS]; 9 10size_t map_bias; 11size_t map_misc_offset; 12size_t arena_maxrun; /* Max run size for arenas. */ 13size_t arena_maxclass; /* Max size class for arenas. */ 14unsigned nlclasses; /* Number of large size classes. */ 15unsigned nhclasses; /* Number of huge size classes. */ 16 17/******************************************************************************/ 18/* 19 * Function prototypes for static functions that are referenced prior to 20 * definition. 21 */ 22 23static void arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk); 24static void arena_purge(arena_t *arena, bool all); 25static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, 26 bool cleaned); 27static void arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, 28 arena_run_t *run, arena_bin_t *bin); 29static void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, 30 arena_run_t *run, arena_bin_t *bin); 31 32/******************************************************************************/ 33 34JEMALLOC_INLINE_C size_t 35arena_miscelm_to_bits(arena_chunk_map_misc_t *miscelm) 36{ 37 arena_chunk_t *chunk = CHUNK_ADDR2BASE(miscelm); 38 size_t pageind = arena_miscelm_to_pageind(miscelm); 39 40 return (arena_mapbits_get(chunk, pageind)); 41} 42 43JEMALLOC_INLINE_C int 44arena_run_comp(arena_chunk_map_misc_t *a, arena_chunk_map_misc_t *b) 45{ 46 uintptr_t a_miscelm = (uintptr_t)a; 47 uintptr_t b_miscelm = (uintptr_t)b; 48 49 assert(a != NULL); 50 assert(b != NULL); 51 52 return ((a_miscelm > b_miscelm) - (a_miscelm < b_miscelm)); 53} 54 55/* Generate red-black tree functions. */ 56rb_gen(static UNUSED, arena_run_tree_, arena_run_tree_t, arena_chunk_map_misc_t, 57 rb_link, arena_run_comp) 58 59JEMALLOC_INLINE_C int 60arena_avail_comp(arena_chunk_map_misc_t *a, arena_chunk_map_misc_t *b) 61{ 62 int ret; 63 size_t a_size; 64 size_t b_size = arena_miscelm_to_bits(b) & ~PAGE_MASK; 65 uintptr_t a_miscelm = (uintptr_t)a; 66 uintptr_t b_miscelm = (uintptr_t)b; 67 68 if (a_miscelm & CHUNK_MAP_KEY) 69 a_size = a_miscelm & ~PAGE_MASK; 70 else 71 a_size = arena_miscelm_to_bits(a) & ~PAGE_MASK; 72 73 ret = (a_size > b_size) - (a_size < b_size); 74 if (ret == 0) { 75 if (!(a_miscelm & CHUNK_MAP_KEY)) 76 ret = (a_miscelm > b_miscelm) - (a_miscelm < b_miscelm); 77 else { 78 /* 79 * Treat keys as if they are lower than anything else. 80 */ 81 ret = -1; 82 } 83 } 84 85 return (ret); 86} 87 88/* Generate red-black tree functions. */ 89rb_gen(static UNUSED, arena_avail_tree_, arena_avail_tree_t, 90 arena_chunk_map_misc_t, rb_link, arena_avail_comp) 91 92static void 93arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 94 size_t npages) 95{ 96 97 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 98 LG_PAGE)); 99 arena_avail_tree_insert(&arena->runs_avail, arena_miscelm_get(chunk, 100 pageind)); 101} 102 103static void 104arena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 105 size_t npages) 106{ 107 108 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 109 LG_PAGE)); 110 arena_avail_tree_remove(&arena->runs_avail, arena_miscelm_get(chunk, 111 pageind)); 112} 113 114static void 115arena_run_dirty_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 116 size_t npages) 117{ 118 arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind); 119 120 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 121 LG_PAGE)); 122 assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY); 123 assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == 124 CHUNK_MAP_DIRTY); 125 126 qr_new(miscelm, rd_link); 127 qr_meld(&arena->runs_dirty, miscelm, rd_link); 128 arena->ndirty += npages; 129} 130 131static void 132arena_run_dirty_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 133 size_t npages) 134{ 135 arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind); 136 137 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 138 LG_PAGE)); 139 assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY); 140 assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == 141 CHUNK_MAP_DIRTY); 142 143 qr_remove(miscelm, rd_link); 144 assert(arena->ndirty >= npages); 145 arena->ndirty -= npages; 146} 147 148static size_t 149arena_chunk_dirty_npages(const extent_node_t *node) 150{ 151 152 return (extent_node_size_get(node) >> LG_PAGE); 153} 154 155static void 156arena_chunk_dirty_node_init(extent_node_t *node) 157{ 158 159 qr_new(node, cd_link); 160 qr_new(&node->runs_dirty, rd_link); 161} 162 163static void 164arena_chunk_dirty_insert(arena_chunk_map_misc_t *runs_dirty, 165 extent_node_t *chunks_dirty, extent_node_t *node) 166{ 167 168 qr_meld(chunks_dirty, node, cd_link); 169 qr_meld(runs_dirty, &node->runs_dirty, rd_link); 170} 171 172static void 173arena_chunk_dirty_remove(extent_node_t *node) 174{ 175 176 qr_remove(node, cd_link); 177 qr_remove(&node->runs_dirty, rd_link); 178} 179 180void 181arena_chunk_dirty_maybe_insert(arena_t *arena, extent_node_t *node, bool dirty) 182{ 183 184 arena_chunk_dirty_node_init(node); 185 if (dirty) { 186 arena_chunk_dirty_insert(&arena->runs_dirty, 187 &arena->chunks_dirty, node); 188 arena->ndirty += arena_chunk_dirty_npages(node); 189 } 190} 191 192void 193arena_chunk_dirty_maybe_remove(arena_t *arena, extent_node_t *node, bool dirty) 194{ 195 196 if (dirty) { 197 arena_chunk_dirty_remove(node); 198 assert(arena->ndirty >= arena_chunk_dirty_npages(node)); 199 arena->ndirty -= arena_chunk_dirty_npages(node); 200 } 201} 202 203JEMALLOC_INLINE_C void * 204arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) 205{ 206 void *ret; 207 unsigned regind; 208 arena_chunk_map_misc_t *miscelm; 209 void *rpages; 210 211 assert(run->nfree > 0); 212 assert(!bitmap_full(run->bitmap, &bin_info->bitmap_info)); 213 214 regind = bitmap_sfu(run->bitmap, &bin_info->bitmap_info); 215 miscelm = arena_run_to_miscelm(run); 216 rpages = arena_miscelm_to_rpages(miscelm); 217 ret = (void *)((uintptr_t)rpages + (uintptr_t)bin_info->reg0_offset + 218 (uintptr_t)(bin_info->reg_interval * regind)); 219 run->nfree--; 220 return (ret); 221} 222 223JEMALLOC_INLINE_C void 224arena_run_reg_dalloc(arena_run_t *run, void *ptr) 225{ 226 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 227 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 228 size_t mapbits = arena_mapbits_get(chunk, pageind); 229 index_t binind = arena_ptr_small_binind_get(ptr, mapbits); 230 arena_bin_info_t *bin_info = &arena_bin_info[binind]; 231 unsigned regind = arena_run_regind(run, bin_info, ptr); 232 233 assert(run->nfree < bin_info->nregs); 234 /* Freeing an interior pointer can cause assertion failure. */ 235 assert(((uintptr_t)ptr - 236 ((uintptr_t)arena_miscelm_to_rpages(arena_run_to_miscelm(run)) + 237 (uintptr_t)bin_info->reg0_offset)) % 238 (uintptr_t)bin_info->reg_interval == 0); 239 assert((uintptr_t)ptr >= 240 (uintptr_t)arena_miscelm_to_rpages(arena_run_to_miscelm(run)) + 241 (uintptr_t)bin_info->reg0_offset); 242 /* Freeing an unallocated pointer can cause assertion failure. */ 243 assert(bitmap_get(run->bitmap, &bin_info->bitmap_info, regind)); 244 245 bitmap_unset(run->bitmap, &bin_info->bitmap_info, regind); 246 run->nfree++; 247} 248 249JEMALLOC_INLINE_C void 250arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages) 251{ 252 253 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + 254 (run_ind << LG_PAGE)), (npages << LG_PAGE)); 255 memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0, 256 (npages << LG_PAGE)); 257} 258 259JEMALLOC_INLINE_C void 260arena_run_page_mark_zeroed(arena_chunk_t *chunk, size_t run_ind) 261{ 262 263 JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind 264 << LG_PAGE)), PAGE); 265} 266 267JEMALLOC_INLINE_C void 268arena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) 269{ 270 size_t i; 271 UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE)); 272 273 arena_run_page_mark_zeroed(chunk, run_ind); 274 for (i = 0; i < PAGE / sizeof(size_t); i++) 275 assert(p[i] == 0); 276} 277 278static void 279arena_cactive_update(arena_t *arena, size_t add_pages, size_t sub_pages) 280{ 281 282 if (config_stats) { 283 ssize_t cactive_diff = CHUNK_CEILING((arena->nactive + add_pages 284 - sub_pages) << LG_PAGE) - CHUNK_CEILING(arena->nactive << 285 LG_PAGE); 286 if (cactive_diff != 0) 287 stats_cactive_add(cactive_diff); 288 } 289} 290 291static void 292arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, 293 size_t flag_dirty, size_t need_pages) 294{ 295 size_t total_pages, rem_pages; 296 297 total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >> 298 LG_PAGE; 299 assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) == 300 flag_dirty); 301 assert(need_pages <= total_pages); 302 rem_pages = total_pages - need_pages; 303 304 arena_avail_remove(arena, chunk, run_ind, total_pages); 305 if (flag_dirty != 0) 306 arena_run_dirty_remove(arena, chunk, run_ind, total_pages); 307 arena_cactive_update(arena, need_pages, 0); 308 arena->nactive += need_pages; 309 310 /* Keep track of trailing unused pages for later use. */ 311 if (rem_pages > 0) { 312 if (flag_dirty != 0) { 313 arena_mapbits_unallocated_set(chunk, 314 run_ind+need_pages, (rem_pages << LG_PAGE), 315 flag_dirty); 316 arena_mapbits_unallocated_set(chunk, 317 run_ind+total_pages-1, (rem_pages << LG_PAGE), 318 flag_dirty); 319 arena_run_dirty_insert(arena, chunk, run_ind+need_pages, 320 rem_pages); 321 } else { 322 arena_mapbits_unallocated_set(chunk, run_ind+need_pages, 323 (rem_pages << LG_PAGE), 324 arena_mapbits_unzeroed_get(chunk, 325 run_ind+need_pages)); 326 arena_mapbits_unallocated_set(chunk, 327 run_ind+total_pages-1, (rem_pages << LG_PAGE), 328 arena_mapbits_unzeroed_get(chunk, 329 run_ind+total_pages-1)); 330 } 331 arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages); 332 } 333} 334 335static void 336arena_run_split_large_helper(arena_t *arena, arena_run_t *run, size_t size, 337 bool remove, bool zero) 338{ 339 arena_chunk_t *chunk; 340 arena_chunk_map_misc_t *miscelm; 341 size_t flag_dirty, run_ind, need_pages, i; 342 343 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 344 miscelm = arena_run_to_miscelm(run); 345 run_ind = arena_miscelm_to_pageind(miscelm); 346 flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); 347 need_pages = (size >> LG_PAGE); 348 assert(need_pages > 0); 349 350 if (remove) { 351 arena_run_split_remove(arena, chunk, run_ind, flag_dirty, 352 need_pages); 353 } 354 355 if (zero) { 356 if (flag_dirty == 0) { 357 /* 358 * The run is clean, so some pages may be zeroed (i.e. 359 * never before touched). 360 */ 361 for (i = 0; i < need_pages; i++) { 362 if (arena_mapbits_unzeroed_get(chunk, run_ind+i) 363 != 0) 364 arena_run_zero(chunk, run_ind+i, 1); 365 else if (config_debug) { 366 arena_run_page_validate_zeroed(chunk, 367 run_ind+i); 368 } else { 369 arena_run_page_mark_zeroed(chunk, 370 run_ind+i); 371 } 372 } 373 } else { 374 /* The run is dirty, so all pages must be zeroed. */ 375 arena_run_zero(chunk, run_ind, need_pages); 376 } 377 } else { 378 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + 379 (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); 380 } 381 382 /* 383 * Set the last element first, in case the run only contains one page 384 * (i.e. both statements set the same element). 385 */ 386 arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0, flag_dirty); 387 arena_mapbits_large_set(chunk, run_ind, size, flag_dirty); 388} 389 390static void 391arena_run_split_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) 392{ 393 394 arena_run_split_large_helper(arena, run, size, true, zero); 395} 396 397static void 398arena_run_init_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) 399{ 400 401 arena_run_split_large_helper(arena, run, size, false, zero); 402} 403 404static void 405arena_run_split_small(arena_t *arena, arena_run_t *run, size_t size, 406 index_t binind) 407{ 408 arena_chunk_t *chunk; 409 arena_chunk_map_misc_t *miscelm; 410 size_t flag_dirty, run_ind, need_pages, i; 411 412 assert(binind != BININD_INVALID); 413 414 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 415 miscelm = arena_run_to_miscelm(run); 416 run_ind = arena_miscelm_to_pageind(miscelm); 417 flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); 418 need_pages = (size >> LG_PAGE); 419 assert(need_pages > 0); 420 421 arena_run_split_remove(arena, chunk, run_ind, flag_dirty, need_pages); 422 423 for (i = 0; i < need_pages; i++) { 424 arena_mapbits_small_set(chunk, run_ind+i, i, binind, 0); 425 if (config_debug && flag_dirty == 0 && 426 arena_mapbits_unzeroed_get(chunk, run_ind+i) == 0) 427 arena_run_page_validate_zeroed(chunk, run_ind+i); 428 } 429 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + 430 (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); 431} 432 433static arena_chunk_t * 434arena_chunk_init_spare(arena_t *arena) 435{ 436 arena_chunk_t *chunk; 437 438 assert(arena->spare != NULL); 439 440 chunk = arena->spare; 441 arena->spare = NULL; 442 443 assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); 444 assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); 445 assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == 446 arena_maxrun); 447 assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == 448 arena_maxrun); 449 assert(arena_mapbits_dirty_get(chunk, map_bias) == 450 arena_mapbits_dirty_get(chunk, chunk_npages-1)); 451 452 return (chunk); 453} 454 455static arena_chunk_t * 456arena_chunk_alloc_internal(arena_t *arena, bool *zero) 457{ 458 arena_chunk_t *chunk; 459 chunk_alloc_t *chunk_alloc; 460 chunk_dalloc_t *chunk_dalloc; 461 462 chunk_alloc = arena->chunk_alloc; 463 chunk_dalloc = arena->chunk_dalloc; 464 malloc_mutex_unlock(&arena->lock); 465 chunk = (arena_chunk_t *)chunk_alloc_arena(chunk_alloc, chunk_dalloc, 466 arena->ind, NULL, chunksize, chunksize, zero); 467 if (chunk != NULL) { 468 extent_node_arena_set(&chunk->node, arena); 469 extent_node_addr_set(&chunk->node, chunk); 470 extent_node_size_set(&chunk->node, chunksize); 471 extent_node_achunk_set(&chunk->node, true); 472 if (chunk_register(chunk, &chunk->node)) { 473 chunk_dalloc((void *)chunk, chunksize, arena->ind); 474 chunk = NULL; 475 } 476 } 477 malloc_mutex_lock(&arena->lock); 478 if (config_stats && chunk != NULL) { 479 arena->stats.mapped += chunksize; 480 arena->stats.metadata_mapped += (map_bias << LG_PAGE); 481 } 482 483 return (chunk); 484} 485 486static arena_chunk_t * 487arena_chunk_init_hard(arena_t *arena) 488{ 489 arena_chunk_t *chunk; 490 bool zero; 491 size_t unzeroed, i; 492 493 assert(arena->spare == NULL); 494 495 zero = false; 496 chunk = arena_chunk_alloc_internal(arena, &zero); 497 if (chunk == NULL) 498 return (NULL); 499 500 /* 501 * Initialize the map to contain one maximal free untouched run. Mark 502 * the pages as zeroed iff chunk_alloc() returned a zeroed chunk. 503 */ 504 unzeroed = zero ? 0 : CHUNK_MAP_UNZEROED; 505 arena_mapbits_unallocated_set(chunk, map_bias, arena_maxrun, unzeroed); 506 /* 507 * There is no need to initialize the internal page map entries unless 508 * the chunk is not zeroed. 509 */ 510 if (!zero) { 511 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED( 512 (void *)arena_bitselm_get(chunk, map_bias+1), 513 (size_t)((uintptr_t) arena_bitselm_get(chunk, 514 chunk_npages-1) - (uintptr_t)arena_bitselm_get(chunk, 515 map_bias+1))); 516 for (i = map_bias+1; i < chunk_npages-1; i++) 517 arena_mapbits_unzeroed_set(chunk, i, unzeroed); 518 } else { 519 JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void 520 *)arena_bitselm_get(chunk, map_bias+1), (size_t)((uintptr_t) 521 arena_bitselm_get(chunk, chunk_npages-1) - 522 (uintptr_t)arena_bitselm_get(chunk, map_bias+1))); 523 if (config_debug) { 524 for (i = map_bias+1; i < chunk_npages-1; i++) { 525 assert(arena_mapbits_unzeroed_get(chunk, i) == 526 unzeroed); 527 } 528 } 529 } 530 arena_mapbits_unallocated_set(chunk, chunk_npages-1, arena_maxrun, 531 unzeroed); 532 533 return (chunk); 534} 535 536static arena_chunk_t * 537arena_chunk_alloc(arena_t *arena) 538{ 539 arena_chunk_t *chunk; 540 541 if (arena->spare != NULL) 542 chunk = arena_chunk_init_spare(arena); 543 else { 544 chunk = arena_chunk_init_hard(arena); 545 if (chunk == NULL) 546 return (NULL); 547 } 548 549 /* Insert the run into the runs_avail tree. */ 550 arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias); 551 552 return (chunk); 553} 554 555static void 556arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk) 557{ 558 559 assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); 560 assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); 561 assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == 562 arena_maxrun); 563 assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == 564 arena_maxrun); 565 assert(arena_mapbits_dirty_get(chunk, map_bias) == 566 arena_mapbits_dirty_get(chunk, chunk_npages-1)); 567 568 /* 569 * Remove run from the runs_avail tree, so that the arena does not use 570 * it. 571 */ 572 arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias); 573 574 if (arena->spare != NULL) { 575 arena_chunk_t *spare = arena->spare; 576 chunk_dalloc_t *chunk_dalloc; 577 578 arena->spare = chunk; 579 if (arena_mapbits_dirty_get(spare, map_bias) != 0) { 580 arena_run_dirty_remove(arena, spare, map_bias, 581 chunk_npages-map_bias); 582 } 583 chunk_dalloc = arena->chunk_dalloc; 584 malloc_mutex_unlock(&arena->lock); 585 chunk_deregister(spare, &spare->node); 586 chunk_dalloc((void *)spare, chunksize, arena->ind); 587 malloc_mutex_lock(&arena->lock); 588 if (config_stats) { 589 arena->stats.mapped -= chunksize; 590 arena->stats.metadata_mapped -= (map_bias << LG_PAGE); 591 } 592 } else 593 arena->spare = chunk; 594} 595 596static void 597arena_huge_malloc_stats_update(arena_t *arena, size_t usize) 598{ 599 index_t index = size2index(usize) - nlclasses - NBINS; 600 601 cassert(config_stats); 602 603 arena->stats.nmalloc_huge++; 604 arena->stats.allocated_huge += usize; 605 arena->stats.hstats[index].nmalloc++; 606 arena->stats.hstats[index].curhchunks++; 607} 608 609static void 610arena_huge_malloc_stats_update_undo(arena_t *arena, size_t usize) 611{ 612 index_t index = size2index(usize) - nlclasses - NBINS; 613 614 cassert(config_stats); 615 616 arena->stats.nmalloc_huge--; 617 arena->stats.allocated_huge -= usize; 618 arena->stats.hstats[index].nmalloc--; 619 arena->stats.hstats[index].curhchunks--; 620} 621 622static void 623arena_huge_dalloc_stats_update(arena_t *arena, size_t usize) 624{ 625 index_t index = size2index(usize) - nlclasses - NBINS; 626 627 cassert(config_stats); 628 629 arena->stats.ndalloc_huge++; 630 arena->stats.allocated_huge -= usize; 631 arena->stats.hstats[index].ndalloc++; 632 arena->stats.hstats[index].curhchunks--; 633} 634 635static void 636arena_huge_dalloc_stats_update_undo(arena_t *arena, size_t usize) 637{ 638 index_t index = size2index(usize) - nlclasses - NBINS; 639 640 cassert(config_stats); 641 642 arena->stats.ndalloc_huge--; 643 arena->stats.allocated_huge += usize; 644 arena->stats.hstats[index].ndalloc--; 645 arena->stats.hstats[index].curhchunks++; 646} 647 648static void 649arena_huge_ralloc_stats_update(arena_t *arena, size_t oldsize, size_t usize) 650{ 651 652 arena_huge_dalloc_stats_update(arena, oldsize); 653 arena_huge_malloc_stats_update(arena, usize); 654} 655 656static void 657arena_huge_ralloc_stats_update_undo(arena_t *arena, size_t oldsize, 658 size_t usize) 659{ 660 661 arena_huge_dalloc_stats_update_undo(arena, oldsize); 662 arena_huge_malloc_stats_update_undo(arena, usize); 663} 664 665extent_node_t * 666arena_node_alloc(arena_t *arena) 667{ 668 extent_node_t *node; 669 670 malloc_mutex_lock(&arena->node_cache_mtx); 671 node = ql_last(&arena->node_cache, ql_link); 672 if (node == NULL) { 673 malloc_mutex_unlock(&arena->node_cache_mtx); 674 return (base_alloc(sizeof(extent_node_t))); 675 } 676 ql_tail_remove(&arena->node_cache, extent_node_t, ql_link); 677 malloc_mutex_unlock(&arena->node_cache_mtx); 678 return (node); 679} 680 681void 682arena_node_dalloc(arena_t *arena, extent_node_t *node) 683{ 684 685 malloc_mutex_lock(&arena->node_cache_mtx); 686 ql_elm_new(node, ql_link); 687 ql_tail_insert(&arena->node_cache, node, ql_link); 688 malloc_mutex_unlock(&arena->node_cache_mtx); 689} 690 691void * 692arena_chunk_alloc_huge(arena_t *arena, size_t usize, size_t alignment, 693 bool *zero) 694{ 695 void *ret; 696 chunk_alloc_t *chunk_alloc; 697 chunk_dalloc_t *chunk_dalloc; 698 size_t csize = CHUNK_CEILING(usize); 699 700 malloc_mutex_lock(&arena->lock); 701 chunk_alloc = arena->chunk_alloc; 702 chunk_dalloc = arena->chunk_dalloc; 703 if (config_stats) { 704 /* Optimistically update stats prior to unlocking. */ 705 arena_huge_malloc_stats_update(arena, usize); 706 arena->stats.mapped += usize; 707 } 708 arena->nactive += (usize >> LG_PAGE); 709 malloc_mutex_unlock(&arena->lock); 710 711 ret = chunk_alloc_arena(chunk_alloc, chunk_dalloc, arena->ind, NULL, 712 csize, alignment, zero); 713 if (ret == NULL) { 714 /* Revert optimistic stats updates. */ 715 malloc_mutex_lock(&arena->lock); 716 if (config_stats) { 717 arena_huge_malloc_stats_update_undo(arena, usize); 718 arena->stats.mapped -= usize; 719 } 720 arena->nactive -= (usize >> LG_PAGE); 721 malloc_mutex_unlock(&arena->lock); 722 return (NULL); 723 } 724 725 if (config_stats) 726 stats_cactive_add(usize); 727 728 return (ret); 729} 730 731void 732arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t usize) 733{ 734 chunk_dalloc_t *chunk_dalloc; 735 736 malloc_mutex_lock(&arena->lock); 737 chunk_dalloc = arena->chunk_dalloc; 738 if (config_stats) { 739 arena_huge_dalloc_stats_update(arena, usize); 740 arena->stats.mapped -= usize; 741 stats_cactive_sub(usize); 742 } 743 arena->nactive -= (usize >> LG_PAGE); 744 malloc_mutex_unlock(&arena->lock); 745 chunk_dalloc(chunk, CHUNK_CEILING(usize), arena->ind); 746} 747 748void 749arena_chunk_ralloc_huge_similar(arena_t *arena, void *chunk, size_t oldsize, 750 size_t usize) 751{ 752 753 assert(CHUNK_CEILING(oldsize) == CHUNK_CEILING(usize)); 754 assert(oldsize != usize); 755 756 malloc_mutex_lock(&arena->lock); 757 if (config_stats) 758 arena_huge_ralloc_stats_update(arena, oldsize, usize); 759 if (oldsize < usize) { 760 size_t udiff = usize - oldsize; 761 arena->nactive += udiff >> LG_PAGE; 762 if (config_stats) 763 stats_cactive_add(udiff); 764 } else { 765 size_t udiff = oldsize - usize; 766 arena->nactive -= udiff >> LG_PAGE; 767 if (config_stats) 768 stats_cactive_sub(udiff); 769 } 770 malloc_mutex_unlock(&arena->lock); 771} 772 773void 774arena_chunk_ralloc_huge_shrink(arena_t *arena, void *chunk, size_t oldsize, 775 size_t usize) 776{ 777 chunk_dalloc_t *chunk_dalloc; 778 size_t udiff = oldsize - usize; 779 size_t cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize); 780 781 malloc_mutex_lock(&arena->lock); 782 chunk_dalloc = arena->chunk_dalloc; 783 if (config_stats) { 784 arena_huge_ralloc_stats_update(arena, oldsize, usize); 785 if (cdiff != 0) { 786 arena->stats.mapped -= cdiff; 787 stats_cactive_sub(udiff); 788 } 789 } 790 arena->nactive -= udiff >> LG_PAGE; 791 malloc_mutex_unlock(&arena->lock); 792 if (cdiff != 0) { 793 chunk_dalloc((void *)((uintptr_t)chunk + CHUNK_CEILING(usize)), 794 cdiff, arena->ind); 795 } 796} 797 798bool 799arena_chunk_ralloc_huge_expand(arena_t *arena, void *chunk, size_t oldsize, 800 size_t usize, bool *zero) 801{ 802 chunk_alloc_t *chunk_alloc; 803 chunk_dalloc_t *chunk_dalloc; 804 size_t udiff = usize - oldsize; 805 size_t cdiff = CHUNK_CEILING(usize) - CHUNK_CEILING(oldsize); 806 807 malloc_mutex_lock(&arena->lock); 808 chunk_alloc = arena->chunk_alloc; 809 chunk_dalloc = arena->chunk_dalloc; 810 if (config_stats) { 811 /* Optimistically update stats prior to unlocking. */ 812 arena_huge_ralloc_stats_update(arena, oldsize, usize); 813 arena->stats.mapped += cdiff; 814 } 815 arena->nactive += (udiff >> LG_PAGE); 816 malloc_mutex_unlock(&arena->lock); 817 818 if (chunk_alloc_arena(chunk_alloc, chunk_dalloc, arena->ind, 819 (void *)((uintptr_t)chunk + CHUNK_CEILING(oldsize)), cdiff, 820 chunksize, zero) == NULL) { 821 /* Revert optimistic stats updates. */ 822 malloc_mutex_lock(&arena->lock); 823 if (config_stats) { 824 arena_huge_ralloc_stats_update_undo(arena, 825 oldsize, usize); 826 arena->stats.mapped -= cdiff; 827 } 828 arena->nactive -= (udiff >> LG_PAGE); 829 malloc_mutex_unlock(&arena->lock); 830 return (true); 831 } 832 833 if (config_stats) 834 stats_cactive_add(udiff); 835 836 return (false); 837} 838 839static arena_run_t * 840arena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero) 841{ 842 arena_chunk_map_misc_t *miscelm; 843 arena_chunk_map_misc_t *key; 844 845 key = (arena_chunk_map_misc_t *)(size | CHUNK_MAP_KEY); 846 miscelm = arena_avail_tree_nsearch(&arena->runs_avail, key); 847 if (miscelm != NULL) { 848 arena_run_t *run = &miscelm->run; 849 arena_run_split_large(arena, &miscelm->run, size, zero); 850 return (run); 851 } 852 853 return (NULL); 854} 855 856static arena_run_t * 857arena_run_alloc_large(arena_t *arena, size_t size, bool zero) 858{ 859 arena_chunk_t *chunk; 860 arena_run_t *run; 861 862 assert(size <= arena_maxrun); 863 assert((size & PAGE_MASK) == 0); 864 865 /* Search the arena's chunks for the lowest best fit. */ 866 run = arena_run_alloc_large_helper(arena, size, zero); 867 if (run != NULL) 868 return (run); 869 870 /* 871 * No usable runs. Create a new chunk from which to allocate the run. 872 */ 873 chunk = arena_chunk_alloc(arena); 874 if (chunk != NULL) { 875 run = &arena_miscelm_get(chunk, map_bias)->run; 876 arena_run_split_large(arena, run, size, zero); 877 return (run); 878 } 879 880 /* 881 * arena_chunk_alloc() failed, but another thread may have made 882 * sufficient memory available while this one dropped arena->lock in 883 * arena_chunk_alloc(), so search one more time. 884 */ 885 return (arena_run_alloc_large_helper(arena, size, zero)); 886} 887 888static arena_run_t * 889arena_run_alloc_small_helper(arena_t *arena, size_t size, index_t binind) 890{ 891 arena_run_t *run; 892 arena_chunk_map_misc_t *miscelm; 893 arena_chunk_map_misc_t *key; 894 895 key = (arena_chunk_map_misc_t *)(size | CHUNK_MAP_KEY); 896 miscelm = arena_avail_tree_nsearch(&arena->runs_avail, key); 897 if (miscelm != NULL) { 898 run = &miscelm->run; 899 arena_run_split_small(arena, run, size, binind); 900 return (run); 901 } 902 903 return (NULL); 904} 905 906static arena_run_t * 907arena_run_alloc_small(arena_t *arena, size_t size, index_t binind) 908{ 909 arena_chunk_t *chunk; 910 arena_run_t *run; 911 912 assert(size <= arena_maxrun); 913 assert((size & PAGE_MASK) == 0); 914 assert(binind != BININD_INVALID); 915 916 /* Search the arena's chunks for the lowest best fit. */ 917 run = arena_run_alloc_small_helper(arena, size, binind); 918 if (run != NULL) 919 return (run); 920 921 /* 922 * No usable runs. Create a new chunk from which to allocate the run. 923 */ 924 chunk = arena_chunk_alloc(arena); 925 if (chunk != NULL) { 926 run = &arena_miscelm_get(chunk, map_bias)->run; 927 arena_run_split_small(arena, run, size, binind); 928 return (run); 929 } 930 931 /* 932 * arena_chunk_alloc() failed, but another thread may have made 933 * sufficient memory available while this one dropped arena->lock in 934 * arena_chunk_alloc(), so search one more time. 935 */ 936 return (arena_run_alloc_small_helper(arena, size, binind)); 937} 938 939JEMALLOC_INLINE_C void 940arena_maybe_purge(arena_t *arena) 941{ 942 size_t threshold; 943 944 /* Don't purge if the option is disabled. */ 945 if (opt_lg_dirty_mult < 0) 946 return; 947 threshold = (arena->nactive >> opt_lg_dirty_mult); 948 threshold = threshold < chunk_npages ? chunk_npages : threshold; 949 /* 950 * Don't purge unless the number of purgeable pages exceeds the 951 * threshold. 952 */ 953 if (arena->ndirty <= threshold) 954 return; 955 956 arena_purge(arena, false); 957} 958 959static size_t 960arena_dirty_count(arena_t *arena) 961{ 962 size_t ndirty = 0; 963 arena_chunk_map_misc_t *runselm; 964 extent_node_t *chunkselm; 965 966 for (runselm = qr_next(&arena->runs_dirty, rd_link), 967 chunkselm = qr_next(&arena->chunks_dirty, cd_link); 968 runselm != &arena->runs_dirty; runselm = qr_next(runselm, 969 rd_link)) { 970 size_t npages; 971 972 if (runselm == &chunkselm->runs_dirty) { 973 npages = extent_node_size_get(chunkselm) >> LG_PAGE; 974 chunkselm = qr_next(chunkselm, cd_link); 975 } else { 976 arena_chunk_t *chunk = (arena_chunk_t 977 *)CHUNK_ADDR2BASE(runselm); 978 size_t pageind = arena_miscelm_to_pageind(runselm); 979 assert(arena_mapbits_allocated_get(chunk, pageind) == 980 0); 981 assert(arena_mapbits_large_get(chunk, pageind) == 0); 982 assert(arena_mapbits_dirty_get(chunk, pageind) != 0); 983 npages = arena_mapbits_unallocated_size_get(chunk, 984 pageind) >> LG_PAGE; 985 } 986 ndirty += npages; 987 } 988 989 return (ndirty); 990} 991 992static size_t 993arena_compute_npurge(arena_t *arena, bool all) 994{ 995 size_t npurge; 996 997 /* 998 * Compute the minimum number of pages that this thread should try to 999 * purge. 1000 */ 1001 if (!all) { 1002 size_t threshold = (arena->nactive >> opt_lg_dirty_mult); 1003 threshold = threshold < chunk_npages ? chunk_npages : threshold; 1004 1005 npurge = arena->ndirty - threshold; 1006 } else 1007 npurge = arena->ndirty; 1008 1009 return (npurge); 1010} 1011 1012static size_t 1013arena_stash_dirty(arena_t *arena, bool all, size_t npurge, 1014 arena_chunk_map_misc_t *purge_runs_sentinel, 1015 extent_node_t *purge_chunks_sentinel) 1016{ 1017 arena_chunk_map_misc_t *runselm, *runselm_next; 1018 extent_node_t *chunkselm; 1019 size_t nstashed = 0; 1020 1021 /* Stash at least npurge pages. */ 1022 for (runselm = qr_next(&arena->runs_dirty, rd_link), 1023 chunkselm = qr_next(&arena->chunks_dirty, cd_link); 1024 runselm != &arena->runs_dirty; runselm = runselm_next) { 1025 size_t npages; 1026 runselm_next = qr_next(runselm, rd_link); 1027 1028 if (runselm == &chunkselm->runs_dirty) { 1029 extent_node_t *chunkselm_next, *tnode; 1030 void *addr; 1031 size_t size; 1032 bool zeroed, zero; 1033 UNUSED void *chunk; 1034 1035 chunkselm_next = qr_next(chunkselm, cd_link); 1036 /* 1037 * Cache contents of chunkselm prior to it being 1038 * destroyed as a side effect of allocating the chunk. 1039 */ 1040 addr = extent_node_addr_get(chunkselm); 1041 size = extent_node_size_get(chunkselm); 1042 zeroed = extent_node_zeroed_get(chunkselm); 1043 /* Allocate. */ 1044 zero = false; 1045 chunk = arena->chunk_alloc(addr, size, chunksize, &zero, 1046 arena->ind); 1047 assert(chunk == addr); 1048 /* 1049 * Create a temporary node to link into the ring of 1050 * stashed allocations. 1051 */ 1052 tnode = arena_node_alloc(arena); 1053 /* 1054 * OOM shouldn't be possible because chunk allocation 1055 * just cached a node. 1056 */ 1057 assert(tnode != NULL); 1058 extent_node_arena_set(tnode, arena); 1059 extent_node_addr_set(tnode, addr); 1060 extent_node_size_set(tnode, size); 1061 extent_node_zeroed_set(tnode, zeroed); 1062 arena_chunk_dirty_node_init(tnode); 1063 /* Stash. */ 1064 arena_chunk_dirty_insert(purge_runs_sentinel, 1065 purge_chunks_sentinel, tnode); 1066 npages = size >> LG_PAGE; 1067 chunkselm = chunkselm_next; 1068 } else { 1069 arena_chunk_t *chunk = 1070 (arena_chunk_t *)CHUNK_ADDR2BASE(runselm); 1071 size_t pageind = arena_miscelm_to_pageind(runselm); 1072 arena_run_t *run = &runselm->run; 1073 size_t run_size = 1074 arena_mapbits_unallocated_size_get(chunk, pageind); 1075 1076 npages = run_size >> LG_PAGE; 1077 1078 assert(pageind + npages <= chunk_npages); 1079 assert(arena_mapbits_dirty_get(chunk, pageind) == 1080 arena_mapbits_dirty_get(chunk, pageind+npages-1)); 1081 1082 /* 1083 * If purging the spare chunk's run, make it available 1084 * prior to allocation. 1085 */ 1086 if (chunk == arena->spare) 1087 arena_chunk_alloc(arena); 1088 1089 /* Temporarily allocate the free dirty run. */ 1090 arena_run_split_large(arena, run, run_size, false); 1091 /* Append to purge_runs for later processing. */ 1092 if (false) 1093 qr_new(runselm, rd_link); /* Redundant. */ 1094 else { 1095 assert(qr_next(runselm, rd_link) == runselm); 1096 assert(qr_prev(runselm, rd_link) == runselm); 1097 } 1098 qr_meld(purge_runs_sentinel, runselm, rd_link); 1099 } 1100 1101 nstashed += npages; 1102 if (!all && nstashed >= npurge) 1103 break; 1104 } 1105 1106 return (nstashed); 1107} 1108 1109static size_t 1110arena_purge_stashed(arena_t *arena, arena_chunk_map_misc_t *purge_runs_sentinel, 1111 extent_node_t *purge_chunks_sentinel) 1112{ 1113 size_t npurged, nmadvise; 1114 arena_chunk_map_misc_t *runselm; 1115 extent_node_t *chunkselm; 1116 1117 if (config_stats) 1118 nmadvise = 0; 1119 npurged = 0; 1120 1121 malloc_mutex_unlock(&arena->lock); 1122 for (runselm = qr_next(purge_runs_sentinel, rd_link), 1123 chunkselm = qr_next(purge_chunks_sentinel, cd_link); 1124 runselm != purge_runs_sentinel; runselm = qr_next(runselm, 1125 rd_link)) { 1126 size_t npages; 1127 1128 if (runselm == &chunkselm->runs_dirty) { 1129 size_t size = extent_node_size_get(chunkselm); 1130 1131 pages_purge(extent_node_addr_get(chunkselm), size); 1132 npages = size >> LG_PAGE; 1133 chunkselm = qr_next(chunkselm, cd_link); 1134 } else { 1135 arena_chunk_t *chunk; 1136 size_t pageind, run_size, flag_unzeroed, i; 1137 bool unzeroed; 1138 1139 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(runselm); 1140 pageind = arena_miscelm_to_pageind(runselm); 1141 run_size = arena_mapbits_large_size_get(chunk, pageind); 1142 npages = run_size >> LG_PAGE; 1143 1144 assert(pageind + npages <= chunk_npages); 1145 unzeroed = pages_purge((void *)((uintptr_t)chunk + 1146 (pageind << LG_PAGE)), run_size); 1147 flag_unzeroed = unzeroed ? CHUNK_MAP_UNZEROED : 0; 1148 1149 /* 1150 * Set the unzeroed flag for all pages, now that 1151 * pages_purge() has returned whether the pages were 1152 * zeroed as a side effect of purging. This chunk map 1153 * modification is safe even though the arena mutex 1154 * isn't currently owned by this thread, because the run 1155 * is marked as allocated, thus protecting it from being 1156 * modified by any other thread. As long as these 1157 * writes don't perturb the first and last elements' 1158 * CHUNK_MAP_ALLOCATED bits, behavior is well defined. 1159 */ 1160 for (i = 0; i < npages; i++) { 1161 arena_mapbits_unzeroed_set(chunk, pageind+i, 1162 flag_unzeroed); 1163 } 1164 } 1165 1166 npurged += npages; 1167 if (config_stats) 1168 nmadvise++; 1169 } 1170 malloc_mutex_lock(&arena->lock); 1171 1172 if (config_stats) { 1173 arena->stats.nmadvise += nmadvise; 1174 arena->stats.purged += npurged; 1175 } 1176 1177 return (npurged); 1178} 1179 1180static void 1181arena_unstash_purged(arena_t *arena, 1182 arena_chunk_map_misc_t *purge_runs_sentinel, 1183 extent_node_t *purge_chunks_sentinel) 1184{ 1185 arena_chunk_map_misc_t *runselm, *runselm_next; 1186 extent_node_t *chunkselm; 1187 1188 /* Deallocate runs. */ 1189 for (runselm = qr_next(purge_runs_sentinel, rd_link), 1190 chunkselm = qr_next(purge_chunks_sentinel, cd_link); 1191 runselm != purge_runs_sentinel; runselm = runselm_next) { 1192 runselm_next = qr_next(runselm, rd_link); 1193 if (runselm == &chunkselm->runs_dirty) { 1194 extent_node_t *chunkselm_next = qr_next(chunkselm, 1195 cd_link); 1196 arena_chunk_dirty_remove(chunkselm); 1197 chunk_unmap(arena, extent_node_addr_get(chunkselm), 1198 extent_node_size_get(chunkselm)); 1199 arena_node_dalloc(arena, chunkselm); 1200 chunkselm = chunkselm_next; 1201 } else { 1202 arena_run_t *run = &runselm->run; 1203 qr_remove(runselm, rd_link); 1204 arena_run_dalloc(arena, run, false, true); 1205 } 1206 } 1207} 1208 1209void 1210arena_purge(arena_t *arena, bool all) 1211{ 1212 size_t npurge, npurgeable, npurged; 1213 arena_chunk_map_misc_t purge_runs_sentinel; 1214 extent_node_t purge_chunks_sentinel; 1215 1216 /* 1217 * Calls to arena_dirty_count() are disabled even for debug builds 1218 * because overhead grows nonlinearly as memory usage increases. 1219 */ 1220 if (false && config_debug) { 1221 size_t ndirty = arena_dirty_count(arena); 1222 assert(ndirty == arena->ndirty); 1223 } 1224 assert((arena->nactive >> opt_lg_dirty_mult) < arena->ndirty || all); 1225 1226 if (config_stats) 1227 arena->stats.npurge++; 1228 1229 npurge = arena_compute_npurge(arena, all); 1230 qr_new(&purge_runs_sentinel, rd_link); 1231 arena_chunk_dirty_node_init(&purge_chunks_sentinel); 1232 1233 npurgeable = arena_stash_dirty(arena, all, npurge, &purge_runs_sentinel, 1234 &purge_chunks_sentinel); 1235 assert(npurgeable >= npurge); 1236 npurged = arena_purge_stashed(arena, &purge_runs_sentinel, 1237 &purge_chunks_sentinel); 1238 assert(npurged == npurgeable); 1239 arena_unstash_purged(arena, &purge_runs_sentinel, 1240 &purge_chunks_sentinel); 1241} 1242 1243void 1244arena_purge_all(arena_t *arena) 1245{ 1246 1247 malloc_mutex_lock(&arena->lock); 1248 arena_purge(arena, true); 1249 malloc_mutex_unlock(&arena->lock); 1250} 1251 1252static void 1253arena_run_coalesce(arena_t *arena, arena_chunk_t *chunk, size_t *p_size, 1254 size_t *p_run_ind, size_t *p_run_pages, size_t flag_dirty) 1255{ 1256 size_t size = *p_size; 1257 size_t run_ind = *p_run_ind; 1258 size_t run_pages = *p_run_pages; 1259 1260 /* Try to coalesce forward. */ 1261 if (run_ind + run_pages < chunk_npages && 1262 arena_mapbits_allocated_get(chunk, run_ind+run_pages) == 0 && 1263 arena_mapbits_dirty_get(chunk, run_ind+run_pages) == flag_dirty) { 1264 size_t nrun_size = arena_mapbits_unallocated_size_get(chunk, 1265 run_ind+run_pages); 1266 size_t nrun_pages = nrun_size >> LG_PAGE; 1267 1268 /* 1269 * Remove successor from runs_avail; the coalesced run is 1270 * inserted later. 1271 */ 1272 assert(arena_mapbits_unallocated_size_get(chunk, 1273 run_ind+run_pages+nrun_pages-1) == nrun_size); 1274 assert(arena_mapbits_dirty_get(chunk, 1275 run_ind+run_pages+nrun_pages-1) == flag_dirty); 1276 arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages); 1277 1278 /* 1279 * If the successor is dirty, remove it from the set of dirty 1280 * pages. 1281 */ 1282 if (flag_dirty != 0) { 1283 arena_run_dirty_remove(arena, chunk, run_ind+run_pages, 1284 nrun_pages); 1285 } 1286 1287 size += nrun_size; 1288 run_pages += nrun_pages; 1289 1290 arena_mapbits_unallocated_size_set(chunk, run_ind, size); 1291 arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, 1292 size); 1293 } 1294 1295 /* Try to coalesce backward. */ 1296 if (run_ind > map_bias && arena_mapbits_allocated_get(chunk, 1297 run_ind-1) == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) == 1298 flag_dirty) { 1299 size_t prun_size = arena_mapbits_unallocated_size_get(chunk, 1300 run_ind-1); 1301 size_t prun_pages = prun_size >> LG_PAGE; 1302 1303 run_ind -= prun_pages; 1304 1305 /* 1306 * Remove predecessor from runs_avail; the coalesced run is 1307 * inserted later. 1308 */ 1309 assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == 1310 prun_size); 1311 assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty); 1312 arena_avail_remove(arena, chunk, run_ind, prun_pages); 1313 1314 /* 1315 * If the predecessor is dirty, remove it from the set of dirty 1316 * pages. 1317 */ 1318 if (flag_dirty != 0) { 1319 arena_run_dirty_remove(arena, chunk, run_ind, 1320 prun_pages); 1321 } 1322 1323 size += prun_size; 1324 run_pages += prun_pages; 1325 1326 arena_mapbits_unallocated_size_set(chunk, run_ind, size); 1327 arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, 1328 size); 1329 } 1330 1331 *p_size = size; 1332 *p_run_ind = run_ind; 1333 *p_run_pages = run_pages; 1334} 1335 1336static void 1337arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) 1338{ 1339 arena_chunk_t *chunk; 1340 arena_chunk_map_misc_t *miscelm; 1341 size_t size, run_ind, run_pages, flag_dirty; 1342 1343 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1344 miscelm = arena_run_to_miscelm(run); 1345 run_ind = arena_miscelm_to_pageind(miscelm); 1346 assert(run_ind >= map_bias); 1347 assert(run_ind < chunk_npages); 1348 if (arena_mapbits_large_get(chunk, run_ind) != 0) { 1349 size = arena_mapbits_large_size_get(chunk, run_ind); 1350 assert(size == PAGE || 1351 arena_mapbits_large_size_get(chunk, 1352 run_ind+(size>>LG_PAGE)-1) == 0); 1353 } else { 1354 arena_bin_info_t *bin_info = &arena_bin_info[run->binind]; 1355 size = bin_info->run_size; 1356 } 1357 run_pages = (size >> LG_PAGE); 1358 arena_cactive_update(arena, 0, run_pages); 1359 arena->nactive -= run_pages; 1360 1361 /* 1362 * The run is dirty if the caller claims to have dirtied it, as well as 1363 * if it was already dirty before being allocated and the caller 1364 * doesn't claim to have cleaned it. 1365 */ 1366 assert(arena_mapbits_dirty_get(chunk, run_ind) == 1367 arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); 1368 if (!cleaned && arena_mapbits_dirty_get(chunk, run_ind) != 0) 1369 dirty = true; 1370 flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; 1371 1372 /* Mark pages as unallocated in the chunk map. */ 1373 if (dirty) { 1374 arena_mapbits_unallocated_set(chunk, run_ind, size, 1375 CHUNK_MAP_DIRTY); 1376 arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, 1377 CHUNK_MAP_DIRTY); 1378 } else { 1379 arena_mapbits_unallocated_set(chunk, run_ind, size, 1380 arena_mapbits_unzeroed_get(chunk, run_ind)); 1381 arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, 1382 arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1)); 1383 } 1384 1385 arena_run_coalesce(arena, chunk, &size, &run_ind, &run_pages, flag_dirty); 1386 1387 /* Insert into runs_avail, now that coalescing is complete. */ 1388 assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == 1389 arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1)); 1390 assert(arena_mapbits_dirty_get(chunk, run_ind) == 1391 arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); 1392 arena_avail_insert(arena, chunk, run_ind, run_pages); 1393 1394 if (dirty) 1395 arena_run_dirty_insert(arena, chunk, run_ind, run_pages); 1396 1397 /* Deallocate chunk if it is now completely unused. */ 1398 if (size == arena_maxrun) { 1399 assert(run_ind == map_bias); 1400 assert(run_pages == (arena_maxrun >> LG_PAGE)); 1401 arena_chunk_dalloc(arena, chunk); 1402 } 1403 1404 /* 1405 * It is okay to do dirty page processing here even if the chunk was 1406 * deallocated above, since in that case it is the spare. Waiting 1407 * until after possible chunk deallocation to do dirty processing 1408 * allows for an old spare to be fully deallocated, thus decreasing the 1409 * chances of spuriously crossing the dirty page purging threshold. 1410 */ 1411 if (dirty) 1412 arena_maybe_purge(arena); 1413} 1414 1415static void 1416arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1417 size_t oldsize, size_t newsize) 1418{ 1419 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); 1420 size_t pageind = arena_miscelm_to_pageind(miscelm); 1421 size_t head_npages = (oldsize - newsize) >> LG_PAGE; 1422 size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); 1423 1424 assert(oldsize > newsize); 1425 1426 /* 1427 * Update the chunk map so that arena_run_dalloc() can treat the 1428 * leading run as separately allocated. Set the last element of each 1429 * run first, in case of single-page runs. 1430 */ 1431 assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); 1432 arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty); 1433 arena_mapbits_large_set(chunk, pageind, oldsize-newsize, flag_dirty); 1434 1435 if (config_debug) { 1436 UNUSED size_t tail_npages = newsize >> LG_PAGE; 1437 assert(arena_mapbits_large_size_get(chunk, 1438 pageind+head_npages+tail_npages-1) == 0); 1439 assert(arena_mapbits_dirty_get(chunk, 1440 pageind+head_npages+tail_npages-1) == flag_dirty); 1441 } 1442 arena_mapbits_large_set(chunk, pageind+head_npages, newsize, 1443 flag_dirty); 1444 1445 arena_run_dalloc(arena, run, false, false); 1446} 1447 1448static void 1449arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1450 size_t oldsize, size_t newsize, bool dirty) 1451{ 1452 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); 1453 size_t pageind = arena_miscelm_to_pageind(miscelm); 1454 size_t head_npages = newsize >> LG_PAGE; 1455 size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); 1456 arena_chunk_map_misc_t *tail_miscelm; 1457 arena_run_t *tail_run; 1458 1459 assert(oldsize > newsize); 1460 1461 /* 1462 * Update the chunk map so that arena_run_dalloc() can treat the 1463 * trailing run as separately allocated. Set the last element of each 1464 * run first, in case of single-page runs. 1465 */ 1466 assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); 1467 arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty); 1468 arena_mapbits_large_set(chunk, pageind, newsize, flag_dirty); 1469 1470 if (config_debug) { 1471 UNUSED size_t tail_npages = (oldsize - newsize) >> LG_PAGE; 1472 assert(arena_mapbits_large_size_get(chunk, 1473 pageind+head_npages+tail_npages-1) == 0); 1474 assert(arena_mapbits_dirty_get(chunk, 1475 pageind+head_npages+tail_npages-1) == flag_dirty); 1476 } 1477 arena_mapbits_large_set(chunk, pageind+head_npages, oldsize-newsize, 1478 flag_dirty); 1479 1480 tail_miscelm = arena_miscelm_get(chunk, pageind + head_npages); 1481 tail_run = &tail_miscelm->run; 1482 arena_run_dalloc(arena, tail_run, dirty, false); 1483} 1484 1485static arena_run_t * 1486arena_bin_runs_first(arena_bin_t *bin) 1487{ 1488 arena_chunk_map_misc_t *miscelm = arena_run_tree_first(&bin->runs); 1489 if (miscelm != NULL) 1490 return (&miscelm->run); 1491 1492 return (NULL); 1493} 1494 1495static void 1496arena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run) 1497{ 1498 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); 1499 1500 assert(arena_run_tree_search(&bin->runs, miscelm) == NULL); 1501 1502 arena_run_tree_insert(&bin->runs, miscelm); 1503} 1504 1505static void 1506arena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run) 1507{ 1508 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); 1509 1510 assert(arena_run_tree_search(&bin->runs, miscelm) != NULL); 1511 1512 arena_run_tree_remove(&bin->runs, miscelm); 1513} 1514 1515static arena_run_t * 1516arena_bin_nonfull_run_tryget(arena_bin_t *bin) 1517{ 1518 arena_run_t *run = arena_bin_runs_first(bin); 1519 if (run != NULL) { 1520 arena_bin_runs_remove(bin, run); 1521 if (config_stats) 1522 bin->stats.reruns++; 1523 } 1524 return (run); 1525} 1526 1527static arena_run_t * 1528arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) 1529{ 1530 arena_run_t *run; 1531 index_t binind; 1532 arena_bin_info_t *bin_info; 1533 1534 /* Look for a usable run. */ 1535 run = arena_bin_nonfull_run_tryget(bin); 1536 if (run != NULL) 1537 return (run); 1538 /* No existing runs have any space available. */ 1539 1540 binind = arena_bin_index(arena, bin); 1541 bin_info = &arena_bin_info[binind]; 1542 1543 /* Allocate a new run. */ 1544 malloc_mutex_unlock(&bin->lock); 1545 /******************************/ 1546 malloc_mutex_lock(&arena->lock); 1547 run = arena_run_alloc_small(arena, bin_info->run_size, binind); 1548 if (run != NULL) { 1549 /* Initialize run internals. */ 1550 run->binind = binind; 1551 run->nfree = bin_info->nregs; 1552 bitmap_init(run->bitmap, &bin_info->bitmap_info); 1553 } 1554 malloc_mutex_unlock(&arena->lock); 1555 /********************************/ 1556 malloc_mutex_lock(&bin->lock); 1557 if (run != NULL) { 1558 if (config_stats) { 1559 bin->stats.nruns++; 1560 bin->stats.curruns++; 1561 } 1562 return (run); 1563 } 1564 1565 /* 1566 * arena_run_alloc_small() failed, but another thread may have made 1567 * sufficient memory available while this one dropped bin->lock above, 1568 * so search one more time. 1569 */ 1570 run = arena_bin_nonfull_run_tryget(bin); 1571 if (run != NULL) 1572 return (run); 1573 1574 return (NULL); 1575} 1576 1577/* Re-fill bin->runcur, then call arena_run_reg_alloc(). */ 1578static void * 1579arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) 1580{ 1581 void *ret; 1582 index_t binind; 1583 arena_bin_info_t *bin_info; 1584 arena_run_t *run; 1585 1586 binind = arena_bin_index(arena, bin); 1587 bin_info = &arena_bin_info[binind]; 1588 bin->runcur = NULL; 1589 run = arena_bin_nonfull_run_get(arena, bin); 1590 if (bin->runcur != NULL && bin->runcur->nfree > 0) { 1591 /* 1592 * Another thread updated runcur while this one ran without the 1593 * bin lock in arena_bin_nonfull_run_get(). 1594 */ 1595 assert(bin->runcur->nfree > 0); 1596 ret = arena_run_reg_alloc(bin->runcur, bin_info); 1597 if (run != NULL) { 1598 arena_chunk_t *chunk; 1599 1600 /* 1601 * arena_run_alloc_small() may have allocated run, or 1602 * it may have pulled run from the bin's run tree. 1603 * Therefore it is unsafe to make any assumptions about 1604 * how run has previously been used, and 1605 * arena_bin_lower_run() must be called, as if a region 1606 * were just deallocated from the run. 1607 */ 1608 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1609 if (run->nfree == bin_info->nregs) 1610 arena_dalloc_bin_run(arena, chunk, run, bin); 1611 else 1612 arena_bin_lower_run(arena, chunk, run, bin); 1613 } 1614 return (ret); 1615 } 1616 1617 if (run == NULL) 1618 return (NULL); 1619 1620 bin->runcur = run; 1621 1622 assert(bin->runcur->nfree > 0); 1623 1624 return (arena_run_reg_alloc(bin->runcur, bin_info)); 1625} 1626 1627void 1628arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, index_t binind, 1629 uint64_t prof_accumbytes) 1630{ 1631 unsigned i, nfill; 1632 arena_bin_t *bin; 1633 arena_run_t *run; 1634 void *ptr; 1635 1636 assert(tbin->ncached == 0); 1637 1638 if (config_prof && arena_prof_accum(arena, prof_accumbytes)) 1639 prof_idump(); 1640 bin = &arena->bins[binind]; 1641 malloc_mutex_lock(&bin->lock); 1642 for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> 1643 tbin->lg_fill_div); i < nfill; i++) { 1644 if ((run = bin->runcur) != NULL && run->nfree > 0) 1645 ptr = arena_run_reg_alloc(run, &arena_bin_info[binind]); 1646 else 1647 ptr = arena_bin_malloc_hard(arena, bin); 1648 if (ptr == NULL) { 1649 /* 1650 * OOM. tbin->avail isn't yet filled down to its first 1651 * element, so the successful allocations (if any) must 1652 * be moved to the base of tbin->avail before bailing 1653 * out. 1654 */ 1655 if (i > 0) { 1656 memmove(tbin->avail, &tbin->avail[nfill - i], 1657 i * sizeof(void *)); 1658 } 1659 break; 1660 } 1661 if (config_fill && unlikely(opt_junk_alloc)) { 1662 arena_alloc_junk_small(ptr, &arena_bin_info[binind], 1663 true); 1664 } 1665 /* Insert such that low regions get used first. */ 1666 tbin->avail[nfill - 1 - i] = ptr; 1667 } 1668 if (config_stats) { 1669 bin->stats.nmalloc += i; 1670 bin->stats.nrequests += tbin->tstats.nrequests; 1671 bin->stats.curregs += i; 1672 bin->stats.nfills++; 1673 tbin->tstats.nrequests = 0; 1674 } 1675 malloc_mutex_unlock(&bin->lock); 1676 tbin->ncached = i; 1677} 1678 1679void 1680arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, bool zero) 1681{ 1682 1683 if (zero) { 1684 size_t redzone_size = bin_info->redzone_size; 1685 memset((void *)((uintptr_t)ptr - redzone_size), 0xa5, 1686 redzone_size); 1687 memset((void *)((uintptr_t)ptr + bin_info->reg_size), 0xa5, 1688 redzone_size); 1689 } else { 1690 memset((void *)((uintptr_t)ptr - bin_info->redzone_size), 0xa5, 1691 bin_info->reg_interval); 1692 } 1693} 1694 1695#ifdef JEMALLOC_JET 1696#undef arena_redzone_corruption 1697#define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption_impl) 1698#endif 1699static void 1700arena_redzone_corruption(void *ptr, size_t usize, bool after, 1701 size_t offset, uint8_t byte) 1702{ 1703 1704 malloc_printf("<jemalloc>: Corrupt redzone %zu byte%s %s %p " 1705 "(size %zu), byte=%#x\n", offset, (offset == 1) ? "" : "s", 1706 after ? "after" : "before", ptr, usize, byte); 1707} 1708#ifdef JEMALLOC_JET 1709#undef arena_redzone_corruption 1710#define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption) 1711arena_redzone_corruption_t *arena_redzone_corruption = 1712 JEMALLOC_N(arena_redzone_corruption_impl); 1713#endif 1714 1715static void 1716arena_redzones_validate(void *ptr, arena_bin_info_t *bin_info, bool reset) 1717{ 1718 size_t size = bin_info->reg_size; 1719 size_t redzone_size = bin_info->redzone_size; 1720 size_t i; 1721 bool error = false; 1722 1723 if (opt_junk_alloc) { 1724 for (i = 1; i <= redzone_size; i++) { 1725 uint8_t *byte = (uint8_t *)((uintptr_t)ptr - i); 1726 if (*byte != 0xa5) { 1727 error = true; 1728 arena_redzone_corruption(ptr, size, false, i, *byte); 1729 if (reset) 1730 *byte = 0xa5; 1731 } 1732 } 1733 for (i = 0; i < redzone_size; i++) { 1734 uint8_t *byte = (uint8_t *)((uintptr_t)ptr + size + i); 1735 if (*byte != 0xa5) { 1736 error = true; 1737 arena_redzone_corruption(ptr, size, true, i, *byte); 1738 if (reset) 1739 *byte = 0xa5; 1740 } 1741 } 1742 } 1743 1744 if (opt_abort && error) 1745 abort(); 1746} 1747 1748#ifdef JEMALLOC_JET 1749#undef arena_dalloc_junk_small 1750#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small_impl) 1751#endif 1752void 1753arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info) 1754{ 1755 size_t redzone_size = bin_info->redzone_size; 1756 1757 arena_redzones_validate(ptr, bin_info, false); 1758 memset((void *)((uintptr_t)ptr - redzone_size), 0x5a, 1759 bin_info->reg_interval); 1760} 1761#ifdef JEMALLOC_JET 1762#undef arena_dalloc_junk_small 1763#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small) 1764arena_dalloc_junk_small_t *arena_dalloc_junk_small = 1765 JEMALLOC_N(arena_dalloc_junk_small_impl); 1766#endif 1767 1768void 1769arena_quarantine_junk_small(void *ptr, size_t usize) 1770{ 1771 index_t binind; 1772 arena_bin_info_t *bin_info; 1773 cassert(config_fill); 1774 assert(opt_junk_free); 1775 assert(opt_quarantine); 1776 assert(usize <= SMALL_MAXCLASS); 1777 1778 binind = size2index(usize); 1779 bin_info = &arena_bin_info[binind]; 1780 arena_redzones_validate(ptr, bin_info, true); 1781} 1782 1783void * 1784arena_malloc_small(arena_t *arena, size_t size, bool zero) 1785{ 1786 void *ret; 1787 arena_bin_t *bin; 1788 arena_run_t *run; 1789 index_t binind; 1790 1791 binind = size2index(size); 1792 assert(binind < NBINS); 1793 bin = &arena->bins[binind]; 1794 size = index2size(binind); 1795 1796 malloc_mutex_lock(&bin->lock); 1797 if ((run = bin->runcur) != NULL && run->nfree > 0) 1798 ret = arena_run_reg_alloc(run, &arena_bin_info[binind]); 1799 else 1800 ret = arena_bin_malloc_hard(arena, bin); 1801 1802 if (ret == NULL) { 1803 malloc_mutex_unlock(&bin->lock); 1804 return (NULL); 1805 } 1806 1807 if (config_stats) { 1808 bin->stats.nmalloc++; 1809 bin->stats.nrequests++; 1810 bin->stats.curregs++; 1811 } 1812 malloc_mutex_unlock(&bin->lock); 1813 if (config_prof && !isthreaded && arena_prof_accum(arena, size)) 1814 prof_idump(); 1815 1816 if (!zero) { 1817 if (config_fill) { 1818 if (unlikely(opt_junk_alloc)) { 1819 arena_alloc_junk_small(ret, 1820 &arena_bin_info[binind], false); 1821 } else if (unlikely(opt_zero)) 1822 memset(ret, 0, size); 1823 } 1824 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, size); 1825 } else { 1826 if (config_fill && unlikely(opt_junk_alloc)) { 1827 arena_alloc_junk_small(ret, &arena_bin_info[binind], 1828 true); 1829 } 1830 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, size); 1831 memset(ret, 0, size); 1832 } 1833 1834 return (ret); 1835} 1836 1837void * 1838arena_malloc_large(arena_t *arena, size_t size, bool zero) 1839{ 1840 void *ret; 1841 size_t usize; 1842 arena_run_t *run; 1843 arena_chunk_map_misc_t *miscelm; 1844 UNUSED bool idump; 1845 1846 /* Large allocation. */ 1847 usize = s2u(size); 1848 malloc_mutex_lock(&arena->lock); 1849 run = arena_run_alloc_large(arena, usize, zero); 1850 if (run == NULL) { 1851 malloc_mutex_unlock(&arena->lock); 1852 return (NULL); 1853 } 1854 miscelm = arena_run_to_miscelm(run); 1855 ret = arena_miscelm_to_rpages(miscelm); 1856 if (config_stats) { 1857 index_t index = size2index(usize) - NBINS; 1858 1859 arena->stats.nmalloc_large++; 1860 arena->stats.nrequests_large++; 1861 arena->stats.allocated_large += usize; 1862 arena->stats.lstats[index].nmalloc++; 1863 arena->stats.lstats[index].nrequests++; 1864 arena->stats.lstats[index].curruns++; 1865 } 1866 if (config_prof) 1867 idump = arena_prof_accum_locked(arena, usize); 1868 malloc_mutex_unlock(&arena->lock); 1869 if (config_prof && idump) 1870 prof_idump(); 1871 1872 if (!zero) { 1873 if (config_fill) { 1874 if (unlikely(opt_junk_alloc)) 1875 memset(ret, 0xa5, usize); 1876 else if (unlikely(opt_zero)) 1877 memset(ret, 0, usize); 1878 } 1879 } 1880 1881 return (ret); 1882} 1883 1884/* Only handles large allocations that require more than page alignment. */ 1885static void * 1886arena_palloc_large(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment, 1887 bool zero) 1888{ 1889 void *ret; 1890 size_t alloc_size, leadsize, trailsize; 1891 arena_run_t *run; 1892 arena_chunk_t *chunk; 1893 arena_chunk_map_misc_t *miscelm; 1894 void *rpages; 1895 1896 assert((size & PAGE_MASK) == 0); 1897 1898 arena = arena_choose(tsd, arena); 1899 if (unlikely(arena == NULL)) 1900 return (NULL); 1901 1902 alignment = PAGE_CEILING(alignment); 1903 alloc_size = size + alignment - PAGE; 1904 1905 malloc_mutex_lock(&arena->lock); 1906 run = arena_run_alloc_large(arena, alloc_size, false); 1907 if (run == NULL) { 1908 malloc_mutex_unlock(&arena->lock); 1909 return (NULL); 1910 } 1911 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1912 miscelm = arena_run_to_miscelm(run); 1913 rpages = arena_miscelm_to_rpages(miscelm); 1914 1915 leadsize = ALIGNMENT_CEILING((uintptr_t)rpages, alignment) - 1916 (uintptr_t)rpages; 1917 assert(alloc_size >= leadsize + size); 1918 trailsize = alloc_size - leadsize - size; 1919 if (leadsize != 0) { 1920 arena_chunk_map_misc_t *head_miscelm = miscelm; 1921 arena_run_t *head_run = run; 1922 1923 miscelm = arena_miscelm_get(chunk, 1924 arena_miscelm_to_pageind(head_miscelm) + (leadsize >> 1925 LG_PAGE)); 1926 run = &miscelm->run; 1927 1928 arena_run_trim_head(arena, chunk, head_run, alloc_size, 1929 alloc_size - leadsize); 1930 } 1931 if (trailsize != 0) { 1932 arena_run_trim_tail(arena, chunk, run, size + trailsize, size, 1933 false); 1934 } 1935 arena_run_init_large(arena, run, size, zero); 1936 ret = arena_miscelm_to_rpages(miscelm); 1937 1938 if (config_stats) { 1939 index_t index = size2index(size) - NBINS; 1940 1941 arena->stats.nmalloc_large++; 1942 arena->stats.nrequests_large++; 1943 arena->stats.allocated_large += size; 1944 arena->stats.lstats[index].nmalloc++; 1945 arena->stats.lstats[index].nrequests++; 1946 arena->stats.lstats[index].curruns++; 1947 } 1948 malloc_mutex_unlock(&arena->lock); 1949 1950 if (config_fill && !zero) { 1951 if (unlikely(opt_junk_alloc)) 1952 memset(ret, 0xa5, size); 1953 else if (unlikely(opt_zero)) 1954 memset(ret, 0, size); 1955 } 1956 return (ret); 1957} 1958 1959void * 1960arena_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, 1961 bool zero, tcache_t *tcache) 1962{ 1963 void *ret; 1964 1965 if (usize <= SMALL_MAXCLASS && alignment < PAGE) 1966 ret = arena_malloc(tsd, arena, usize, zero, tcache); 1967 else { 1968 if (likely(usize <= arena_maxclass)) { 1969 ret = arena_palloc_large(tsd, arena, usize, alignment, 1970 zero); 1971 } else if (likely(alignment <= chunksize)) 1972 ret = huge_malloc(tsd, arena, usize, zero, tcache); 1973 else { 1974 ret = huge_palloc(tsd, arena, usize, alignment, zero, 1975 tcache); 1976 } 1977 } 1978 return (ret); 1979} 1980 1981void 1982arena_prof_promoted(const void *ptr, size_t size) 1983{ 1984 arena_chunk_t *chunk; 1985 size_t pageind; 1986 index_t binind; 1987 1988 cassert(config_prof); 1989 assert(ptr != NULL); 1990 assert(CHUNK_ADDR2BASE(ptr) != ptr); 1991 assert(isalloc(ptr, false) == LARGE_MINCLASS); 1992 assert(isalloc(ptr, true) == LARGE_MINCLASS); 1993 assert(size <= SMALL_MAXCLASS); 1994 1995 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1996 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1997 binind = size2index(size); 1998 assert(binind < NBINS); 1999 arena_mapbits_large_binind_set(chunk, pageind, binind); 2000 2001 assert(isalloc(ptr, false) == LARGE_MINCLASS); 2002 assert(isalloc(ptr, true) == size); 2003} 2004 2005static void 2006arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, 2007 arena_bin_t *bin) 2008{ 2009 2010 /* Dissociate run from bin. */ 2011 if (run == bin->runcur) 2012 bin->runcur = NULL; 2013 else { 2014 index_t binind = arena_bin_index(extent_node_arena_get( 2015 &chunk->node), bin); 2016 arena_bin_info_t *bin_info = &arena_bin_info[binind]; 2017 2018 if (bin_info->nregs != 1) { 2019 /* 2020 * This block's conditional is necessary because if the 2021 * run only contains one region, then it never gets 2022 * inserted into the non-full runs tree. 2023 */ 2024 arena_bin_runs_remove(bin, run); 2025 } 2026 } 2027} 2028 2029static void 2030arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 2031 arena_bin_t *bin) 2032{ 2033 2034 assert(run != bin->runcur); 2035 assert(arena_run_tree_search(&bin->runs, arena_run_to_miscelm(run)) == 2036 NULL); 2037 2038 malloc_mutex_unlock(&bin->lock); 2039 /******************************/ 2040 malloc_mutex_lock(&arena->lock); 2041 arena_run_dalloc(arena, run, true, false); 2042 malloc_mutex_unlock(&arena->lock); 2043 /****************************/ 2044 malloc_mutex_lock(&bin->lock); 2045 if (config_stats) 2046 bin->stats.curruns--; 2047} 2048 2049static void 2050arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 2051 arena_bin_t *bin) 2052{ 2053 2054 /* 2055 * Make sure that if bin->runcur is non-NULL, it refers to the lowest 2056 * non-full run. It is okay to NULL runcur out rather than proactively 2057 * keeping it pointing at the lowest non-full run. 2058 */ 2059 if ((uintptr_t)run < (uintptr_t)bin->runcur) { 2060 /* Switch runcur. */ 2061 if (bin->runcur->nfree > 0) 2062 arena_bin_runs_insert(bin, bin->runcur); 2063 bin->runcur = run; 2064 if (config_stats) 2065 bin->stats.reruns++; 2066 } else 2067 arena_bin_runs_insert(bin, run); 2068} 2069 2070static void 2071arena_dalloc_bin_locked_impl(arena_t *arena, arena_chunk_t *chunk, void *ptr, 2072 arena_chunk_map_bits_t *bitselm, bool junked) 2073{ 2074 size_t pageind, rpages_ind; 2075 arena_run_t *run; 2076 arena_bin_t *bin; 2077 arena_bin_info_t *bin_info; 2078 index_t binind; 2079 2080 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 2081 rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind); 2082 run = &arena_miscelm_get(chunk, rpages_ind)->run; 2083 binind = run->binind; 2084 bin = &arena->bins[binind]; 2085 bin_info = &arena_bin_info[binind]; 2086 2087 if (!junked && config_fill && unlikely(opt_junk_free)) 2088 arena_dalloc_junk_small(ptr, bin_info); 2089 2090 arena_run_reg_dalloc(run, ptr); 2091 if (run->nfree == bin_info->nregs) { 2092 arena_dissociate_bin_run(chunk, run, bin); 2093 arena_dalloc_bin_run(arena, chunk, run, bin); 2094 } else if (run->nfree == 1 && run != bin->runcur) 2095 arena_bin_lower_run(arena, chunk, run, bin); 2096 2097 if (config_stats) { 2098 bin->stats.ndalloc++; 2099 bin->stats.curregs--; 2100 } 2101} 2102 2103void 2104arena_dalloc_bin_junked_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, 2105 arena_chunk_map_bits_t *bitselm) 2106{ 2107 2108 arena_dalloc_bin_locked_impl(arena, chunk, ptr, bitselm, true); 2109} 2110 2111void 2112arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, 2113 size_t pageind, arena_chunk_map_bits_t *bitselm) 2114{ 2115 arena_run_t *run; 2116 arena_bin_t *bin; 2117 size_t rpages_ind; 2118 2119 rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind); 2120 run = &arena_miscelm_get(chunk, rpages_ind)->run; 2121 bin = &arena->bins[run->binind]; 2122 malloc_mutex_lock(&bin->lock); 2123 arena_dalloc_bin_locked_impl(arena, chunk, ptr, bitselm, false); 2124 malloc_mutex_unlock(&bin->lock); 2125} 2126 2127void 2128arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, 2129 size_t pageind) 2130{ 2131 arena_chunk_map_bits_t *bitselm; 2132 2133 if (config_debug) { 2134 /* arena_ptr_small_binind_get() does extra sanity checking. */ 2135 assert(arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, 2136 pageind)) != BININD_INVALID); 2137 } 2138 bitselm = arena_bitselm_get(chunk, pageind); 2139 arena_dalloc_bin(arena, chunk, ptr, pageind, bitselm); 2140} 2141 2142#ifdef JEMALLOC_JET 2143#undef arena_dalloc_junk_large 2144#define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large_impl) 2145#endif 2146void 2147arena_dalloc_junk_large(void *ptr, size_t usize) 2148{ 2149 2150 if (config_fill && unlikely(opt_junk_free)) 2151 memset(ptr, 0x5a, usize); 2152} 2153#ifdef JEMALLOC_JET 2154#undef arena_dalloc_junk_large 2155#define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large) 2156arena_dalloc_junk_large_t *arena_dalloc_junk_large = 2157 JEMALLOC_N(arena_dalloc_junk_large_impl); 2158#endif 2159 2160void 2161arena_dalloc_large_locked_impl(arena_t *arena, arena_chunk_t *chunk, 2162 void *ptr, bool junked) 2163{ 2164 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 2165 arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind); 2166 arena_run_t *run = &miscelm->run; 2167 2168 if (config_fill || config_stats) { 2169 size_t usize = arena_mapbits_large_size_get(chunk, pageind); 2170 2171 if (!junked) 2172 arena_dalloc_junk_large(ptr, usize); 2173 if (config_stats) { 2174 index_t index = size2index(usize) - NBINS; 2175 2176 arena->stats.ndalloc_large++; 2177 arena->stats.allocated_large -= usize; 2178 arena->stats.lstats[index].ndalloc++; 2179 arena->stats.lstats[index].curruns--; 2180 } 2181 } 2182 2183 arena_run_dalloc(arena, run, true, false); 2184} 2185 2186void 2187arena_dalloc_large_junked_locked(arena_t *arena, arena_chunk_t *chunk, 2188 void *ptr) 2189{ 2190 2191 arena_dalloc_large_locked_impl(arena, chunk, ptr, true); 2192} 2193 2194void 2195arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr) 2196{ 2197 2198 malloc_mutex_lock(&arena->lock); 2199 arena_dalloc_large_locked_impl(arena, chunk, ptr, false); 2200 malloc_mutex_unlock(&arena->lock); 2201} 2202 2203static void 2204arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, 2205 size_t oldsize, size_t size) 2206{ 2207 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 2208 arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind); 2209 arena_run_t *run = &miscelm->run; 2210 2211 assert(size < oldsize); 2212 2213 /* 2214 * Shrink the run, and make trailing pages available for other 2215 * allocations. 2216 */ 2217 malloc_mutex_lock(&arena->lock); 2218 arena_run_trim_tail(arena, chunk, run, oldsize, size, true); 2219 if (config_stats) { 2220 index_t oldindex = size2index(oldsize) - NBINS; 2221 index_t index = size2index(size) - NBINS; 2222 2223 arena->stats.ndalloc_large++; 2224 arena->stats.allocated_large -= oldsize; 2225 arena->stats.lstats[oldindex].ndalloc++; 2226 arena->stats.lstats[oldindex].curruns--; 2227 2228 arena->stats.nmalloc_large++; 2229 arena->stats.nrequests_large++; 2230 arena->stats.allocated_large += size; 2231 arena->stats.lstats[index].nmalloc++; 2232 arena->stats.lstats[index].nrequests++; 2233 arena->stats.lstats[index].curruns++; 2234 } 2235 malloc_mutex_unlock(&arena->lock); 2236} 2237 2238static bool 2239arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, 2240 size_t oldsize, size_t size, size_t extra, bool zero) 2241{ 2242 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 2243 size_t npages = oldsize >> LG_PAGE; 2244 size_t followsize; 2245 size_t usize_min = s2u(size); 2246 2247 assert(oldsize == arena_mapbits_large_size_get(chunk, pageind)); 2248 2249 /* Try to extend the run. */ 2250 assert(usize_min > oldsize); 2251 malloc_mutex_lock(&arena->lock); 2252 if (pageind + npages < chunk_npages && 2253 arena_mapbits_allocated_get(chunk, pageind+npages) == 0 && 2254 (followsize = arena_mapbits_unallocated_size_get(chunk, 2255 pageind+npages)) >= usize_min - oldsize) { 2256 /* 2257 * The next run is available and sufficiently large. Split the 2258 * following run, then merge the first part with the existing 2259 * allocation. 2260 */ 2261 arena_run_t *run; 2262 size_t flag_dirty, splitsize, usize; 2263 2264 usize = s2u(size + extra); 2265 while (oldsize + followsize < usize) 2266 usize = index2size(size2index(usize)-1); 2267 assert(usize >= usize_min); 2268 splitsize = usize - oldsize; 2269 2270 run = &arena_miscelm_get(chunk, pageind+npages)->run; 2271 arena_run_split_large(arena, run, splitsize, zero); 2272 2273 size = oldsize + splitsize; 2274 npages = size >> LG_PAGE; 2275 2276 /* 2277 * Mark the extended run as dirty if either portion of the run 2278 * was dirty before allocation. This is rather pedantic, 2279 * because there's not actually any sequence of events that 2280 * could cause the resulting run to be passed to 2281 * arena_run_dalloc() with the dirty argument set to false 2282 * (which is when dirty flag consistency would really matter). 2283 */ 2284 flag_dirty = arena_mapbits_dirty_get(chunk, pageind) | 2285 arena_mapbits_dirty_get(chunk, pageind+npages-1); 2286 arena_mapbits_large_set(chunk, pageind, size, flag_dirty); 2287 arena_mapbits_large_set(chunk, pageind+npages-1, 0, flag_dirty); 2288 2289 if (config_stats) { 2290 index_t oldindex = size2index(oldsize) - NBINS; 2291 index_t index = size2index(size) - NBINS; 2292 2293 arena->stats.ndalloc_large++; 2294 arena->stats.allocated_large -= oldsize; 2295 arena->stats.lstats[oldindex].ndalloc++; 2296 arena->stats.lstats[oldindex].curruns--; 2297 2298 arena->stats.nmalloc_large++; 2299 arena->stats.nrequests_large++; 2300 arena->stats.allocated_large += size; 2301 arena->stats.lstats[index].nmalloc++; 2302 arena->stats.lstats[index].nrequests++; 2303 arena->stats.lstats[index].curruns++; 2304 } 2305 malloc_mutex_unlock(&arena->lock); 2306 return (false); 2307 } 2308 malloc_mutex_unlock(&arena->lock); 2309 2310 return (true); 2311} 2312 2313#ifdef JEMALLOC_JET 2314#undef arena_ralloc_junk_large 2315#define arena_ralloc_junk_large JEMALLOC_N(arena_ralloc_junk_large_impl) 2316#endif 2317static void 2318arena_ralloc_junk_large(void *ptr, size_t old_usize, size_t usize) 2319{ 2320 2321 if (config_fill && unlikely(opt_junk_free)) { 2322 memset((void *)((uintptr_t)ptr + usize), 0x5a, 2323 old_usize - usize); 2324 } 2325} 2326#ifdef JEMALLOC_JET 2327#undef arena_ralloc_junk_large 2328#define arena_ralloc_junk_large JEMALLOC_N(arena_ralloc_junk_large) 2329arena_ralloc_junk_large_t *arena_ralloc_junk_large = 2330 JEMALLOC_N(arena_ralloc_junk_large_impl); 2331#endif 2332 2333/* 2334 * Try to resize a large allocation, in order to avoid copying. This will 2335 * always fail if growing an object, and the following run is already in use. 2336 */ 2337static bool 2338arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, 2339 bool zero) 2340{ 2341 size_t usize; 2342 2343 /* Make sure extra can't cause size_t overflow. */ 2344 if (unlikely(extra >= arena_maxclass)) 2345 return (true); 2346 2347 usize = s2u(size + extra); 2348 if (usize == oldsize) { 2349 /* Same size class. */ 2350 return (false); 2351 } else { 2352 arena_chunk_t *chunk; 2353 arena_t *arena; 2354 2355 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 2356 arena = extent_node_arena_get(&chunk->node); 2357 2358 if (usize < oldsize) { 2359 /* Fill before shrinking in order avoid a race. */ 2360 arena_ralloc_junk_large(ptr, oldsize, usize); 2361 arena_ralloc_large_shrink(arena, chunk, ptr, oldsize, 2362 usize); 2363 return (false); 2364 } else { 2365 bool ret = arena_ralloc_large_grow(arena, chunk, ptr, 2366 oldsize, size, extra, zero); 2367 if (config_fill && !ret && !zero) { 2368 if (unlikely(opt_junk_alloc)) { 2369 memset((void *)((uintptr_t)ptr + 2370 oldsize), 0xa5, isalloc(ptr, 2371 config_prof) - oldsize); 2372 } else if (unlikely(opt_zero)) { 2373 memset((void *)((uintptr_t)ptr + 2374 oldsize), 0, isalloc(ptr, 2375 config_prof) - oldsize); 2376 } 2377 } 2378 return (ret); 2379 } 2380 } 2381} 2382 2383bool 2384arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, 2385 bool zero) 2386{ 2387 2388 if (likely(size <= arena_maxclass)) { 2389 /* 2390 * Avoid moving the allocation if the size class can be left the 2391 * same. 2392 */ 2393 if (likely(oldsize <= arena_maxclass)) { 2394 if (oldsize <= SMALL_MAXCLASS) { 2395 assert( 2396 arena_bin_info[size2index(oldsize)].reg_size 2397 == oldsize); 2398 if ((size + extra <= SMALL_MAXCLASS && 2399 size2index(size + extra) == 2400 size2index(oldsize)) || (size <= oldsize && 2401 size + extra >= oldsize)) 2402 return (false); 2403 } else { 2404 assert(size <= arena_maxclass); 2405 if (size + extra > SMALL_MAXCLASS) { 2406 if (!arena_ralloc_large(ptr, oldsize, 2407 size, extra, zero)) 2408 return (false); 2409 } 2410 } 2411 } 2412 2413 /* Reallocation would require a move. */ 2414 return (true); 2415 } else 2416 return (huge_ralloc_no_move(ptr, oldsize, size, extra, zero)); 2417} 2418 2419void * 2420arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, 2421 size_t extra, size_t alignment, bool zero, tcache_t *tcache) 2422{ 2423 void *ret; 2424 2425 if (likely(size <= arena_maxclass)) { 2426 size_t copysize; 2427 2428 /* Try to avoid moving the allocation. */ 2429 if (!arena_ralloc_no_move(ptr, oldsize, size, extra, zero)) 2430 return (ptr); 2431 2432 /* 2433 * size and oldsize are different enough that we need to move 2434 * the object. In that case, fall back to allocating new space 2435 * and copying. 2436 */ 2437 if (alignment != 0) { 2438 size_t usize = sa2u(size + extra, alignment); 2439 if (usize == 0) 2440 return (NULL); 2441 ret = ipalloct(tsd, usize, alignment, zero, tcache, 2442 arena); 2443 } else { 2444 ret = arena_malloc(tsd, arena, size + extra, zero, 2445 tcache); 2446 } 2447 2448 if (ret == NULL) { 2449 if (extra == 0) 2450 return (NULL); 2451 /* Try again, this time without extra. */ 2452 if (alignment != 0) { 2453 size_t usize = sa2u(size, alignment); 2454 if (usize == 0) 2455 return (NULL); 2456 ret = ipalloct(tsd, usize, alignment, zero, 2457 tcache, arena); 2458 } else { 2459 ret = arena_malloc(tsd, arena, size, zero, 2460 tcache); 2461 } 2462 2463 if (ret == NULL) 2464 return (NULL); 2465 } 2466 2467 /* 2468 * Junk/zero-filling were already done by 2469 * ipalloc()/arena_malloc(). 2470 */ 2471 2472 /* 2473 * Copy at most size bytes (not size+extra), since the caller 2474 * has no expectation that the extra bytes will be reliably 2475 * preserved. 2476 */ 2477 copysize = (size < oldsize) ? size : oldsize; 2478 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize); 2479 memcpy(ret, ptr, copysize); 2480 isqalloc(tsd, ptr, oldsize, tcache); 2481 } else { 2482 ret = huge_ralloc(tsd, arena, ptr, oldsize, size, extra, 2483 alignment, zero, tcache); 2484 } 2485 return (ret); 2486} 2487 2488dss_prec_t 2489arena_dss_prec_get(arena_t *arena) 2490{ 2491 dss_prec_t ret; 2492 2493 malloc_mutex_lock(&arena->lock); 2494 ret = arena->dss_prec; 2495 malloc_mutex_unlock(&arena->lock); 2496 return (ret); 2497} 2498 2499bool 2500arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) 2501{ 2502 2503 if (!have_dss) 2504 return (dss_prec != dss_prec_disabled); 2505 malloc_mutex_lock(&arena->lock); 2506 arena->dss_prec = dss_prec; 2507 malloc_mutex_unlock(&arena->lock); 2508 return (false); 2509} 2510 2511void 2512arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, 2513 size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, 2514 malloc_large_stats_t *lstats, malloc_huge_stats_t *hstats) 2515{ 2516 unsigned i; 2517 2518 malloc_mutex_lock(&arena->lock); 2519 *dss = dss_prec_names[arena->dss_prec]; 2520 *nactive += arena->nactive; 2521 *ndirty += arena->ndirty; 2522 2523 astats->mapped += arena->stats.mapped; 2524 astats->npurge += arena->stats.npurge; 2525 astats->nmadvise += arena->stats.nmadvise; 2526 astats->purged += arena->stats.purged; 2527 astats->metadata_mapped += arena->stats.metadata_mapped; 2528 astats->metadata_allocated += arena_metadata_allocated_get(arena); 2529 astats->allocated_large += arena->stats.allocated_large; 2530 astats->nmalloc_large += arena->stats.nmalloc_large; 2531 astats->ndalloc_large += arena->stats.ndalloc_large; 2532 astats->nrequests_large += arena->stats.nrequests_large; 2533 astats->allocated_huge += arena->stats.allocated_huge; 2534 astats->nmalloc_huge += arena->stats.nmalloc_huge; 2535 astats->ndalloc_huge += arena->stats.ndalloc_huge; 2536 2537 for (i = 0; i < nlclasses; i++) { 2538 lstats[i].nmalloc += arena->stats.lstats[i].nmalloc; 2539 lstats[i].ndalloc += arena->stats.lstats[i].ndalloc; 2540 lstats[i].nrequests += arena->stats.lstats[i].nrequests; 2541 lstats[i].curruns += arena->stats.lstats[i].curruns; 2542 } 2543 2544 for (i = 0; i < nhclasses; i++) { 2545 hstats[i].nmalloc += arena->stats.hstats[i].nmalloc; 2546 hstats[i].ndalloc += arena->stats.hstats[i].ndalloc; 2547 hstats[i].curhchunks += arena->stats.hstats[i].curhchunks; 2548 } 2549 malloc_mutex_unlock(&arena->lock); 2550 2551 for (i = 0; i < NBINS; i++) { 2552 arena_bin_t *bin = &arena->bins[i]; 2553 2554 malloc_mutex_lock(&bin->lock); 2555 bstats[i].nmalloc += bin->stats.nmalloc; 2556 bstats[i].ndalloc += bin->stats.ndalloc; 2557 bstats[i].nrequests += bin->stats.nrequests; 2558 bstats[i].curregs += bin->stats.curregs; 2559 if (config_tcache) { 2560 bstats[i].nfills += bin->stats.nfills; 2561 bstats[i].nflushes += bin->stats.nflushes; 2562 } 2563 bstats[i].nruns += bin->stats.nruns; 2564 bstats[i].reruns += bin->stats.reruns; 2565 bstats[i].curruns += bin->stats.curruns; 2566 malloc_mutex_unlock(&bin->lock); 2567 } 2568} 2569 2570arena_t * 2571arena_new(unsigned ind) 2572{ 2573 arena_t *arena; 2574 unsigned i; 2575 arena_bin_t *bin; 2576 2577 /* 2578 * Allocate arena, arena->lstats, and arena->hstats contiguously, mainly 2579 * because there is no way to clean up if base_alloc() OOMs. 2580 */ 2581 if (config_stats) { 2582 arena = (arena_t *)base_alloc(CACHELINE_CEILING(sizeof(arena_t)) 2583 + QUANTUM_CEILING(nlclasses * sizeof(malloc_large_stats_t) + 2584 nhclasses) * sizeof(malloc_huge_stats_t)); 2585 } else 2586 arena = (arena_t *)base_alloc(sizeof(arena_t)); 2587 if (arena == NULL) 2588 return (NULL); 2589 2590 arena->ind = ind; 2591 arena->nthreads = 0; 2592 if (malloc_mutex_init(&arena->lock)) 2593 return (NULL); 2594 2595 if (config_stats) { 2596 memset(&arena->stats, 0, sizeof(arena_stats_t)); 2597 arena->stats.lstats = (malloc_large_stats_t *)((uintptr_t)arena 2598 + CACHELINE_CEILING(sizeof(arena_t))); 2599 memset(arena->stats.lstats, 0, nlclasses * 2600 sizeof(malloc_large_stats_t)); 2601 arena->stats.hstats = (malloc_huge_stats_t *)((uintptr_t)arena 2602 + CACHELINE_CEILING(sizeof(arena_t)) + 2603 QUANTUM_CEILING(nlclasses * sizeof(malloc_large_stats_t))); 2604 memset(arena->stats.hstats, 0, nhclasses * 2605 sizeof(malloc_huge_stats_t)); 2606 if (config_tcache) 2607 ql_new(&arena->tcache_ql); 2608 } 2609 2610 if (config_prof) 2611 arena->prof_accumbytes = 0; 2612 2613 arena->dss_prec = chunk_dss_prec_get(); 2614 2615 arena->spare = NULL; 2616 2617 arena->nactive = 0; 2618 arena->ndirty = 0; 2619 2620 arena_avail_tree_new(&arena->runs_avail); 2621 qr_new(&arena->runs_dirty, rd_link); 2622 qr_new(&arena->chunks_dirty, cd_link); 2623 2624 ql_new(&arena->huge); 2625 if (malloc_mutex_init(&arena->huge_mtx)) 2626 return (NULL); 2627 2628 extent_tree_szad_new(&arena->chunks_szad_dirty); 2629 extent_tree_ad_new(&arena->chunks_ad_dirty); 2630 extent_tree_szad_new(&arena->chunks_szad_mmap); 2631 extent_tree_ad_new(&arena->chunks_ad_mmap); 2632 extent_tree_szad_new(&arena->chunks_szad_dss); 2633 extent_tree_ad_new(&arena->chunks_ad_dss); 2634 if (malloc_mutex_init(&arena->chunks_mtx)) 2635 return (NULL); 2636 ql_new(&arena->node_cache); 2637 if (malloc_mutex_init(&arena->node_cache_mtx)) 2638 return (NULL); 2639 2640 arena->chunk_alloc = chunk_alloc_default; 2641 arena->chunk_dalloc = chunk_dalloc_default; 2642 2643 /* Initialize bins. */ 2644 for (i = 0; i < NBINS; i++) { 2645 bin = &arena->bins[i]; 2646 if (malloc_mutex_init(&bin->lock)) 2647 return (NULL); 2648 bin->runcur = NULL; 2649 arena_run_tree_new(&bin->runs); 2650 if (config_stats) 2651 memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 2652 } 2653 2654 return (arena); 2655} 2656 2657/* 2658 * Calculate bin_info->run_size such that it meets the following constraints: 2659 * 2660 * *) bin_info->run_size <= arena_maxrun 2661 * *) bin_info->nregs <= RUN_MAXREGS 2662 * 2663 * bin_info->nregs and bin_info->reg0_offset are also calculated here, since 2664 * these settings are all interdependent. 2665 */ 2666static void 2667bin_info_run_size_calc(arena_bin_info_t *bin_info) 2668{ 2669 size_t pad_size; 2670 size_t try_run_size, perfect_run_size, actual_run_size; 2671 uint32_t try_nregs, perfect_nregs, actual_nregs; 2672 2673 /* 2674 * Determine redzone size based on minimum alignment and minimum 2675 * redzone size. Add padding to the end of the run if it is needed to 2676 * align the regions. The padding allows each redzone to be half the 2677 * minimum alignment; without the padding, each redzone would have to 2678 * be twice as large in order to maintain alignment. 2679 */ 2680 if (config_fill && unlikely(opt_redzone)) { 2681 size_t align_min = ZU(1) << (jemalloc_ffs(bin_info->reg_size) - 2682 1); 2683 if (align_min <= REDZONE_MINSIZE) { 2684 bin_info->redzone_size = REDZONE_MINSIZE; 2685 pad_size = 0; 2686 } else { 2687 bin_info->redzone_size = align_min >> 1; 2688 pad_size = bin_info->redzone_size; 2689 } 2690 } else { 2691 bin_info->redzone_size = 0; 2692 pad_size = 0; 2693 } 2694 bin_info->reg_interval = bin_info->reg_size + 2695 (bin_info->redzone_size << 1); 2696 2697 /* 2698 * Compute run size under ideal conditions (no redzones, no limit on run 2699 * size). 2700 */ 2701 try_run_size = PAGE; 2702 try_nregs = try_run_size / bin_info->reg_size; 2703 do { 2704 perfect_run_size = try_run_size; 2705 perfect_nregs = try_nregs; 2706 2707 try_run_size += PAGE; 2708 try_nregs = try_run_size / bin_info->reg_size; 2709 } while (perfect_run_size != perfect_nregs * bin_info->reg_size); 2710 assert(perfect_nregs <= RUN_MAXREGS); 2711 2712 actual_run_size = perfect_run_size; 2713 actual_nregs = (actual_run_size - pad_size) / bin_info->reg_interval; 2714 2715 /* 2716 * Redzones can require enough padding that not even a single region can 2717 * fit within the number of pages that would normally be dedicated to a 2718 * run for this size class. Increase the run size until at least one 2719 * region fits. 2720 */ 2721 while (actual_nregs == 0) { 2722 assert(config_fill && unlikely(opt_redzone)); 2723 2724 actual_run_size += PAGE; 2725 actual_nregs = (actual_run_size - pad_size) / 2726 bin_info->reg_interval; 2727 } 2728 2729 /* 2730 * Make sure that the run will fit within an arena chunk. 2731 */ 2732 while (actual_run_size > arena_maxrun) { 2733 actual_run_size -= PAGE; 2734 actual_nregs = (actual_run_size - pad_size) / 2735 bin_info->reg_interval; 2736 } 2737 assert(actual_nregs > 0); 2738 2739 /* Copy final settings. */ 2740 bin_info->run_size = actual_run_size; 2741 bin_info->nregs = actual_nregs; 2742 bin_info->reg0_offset = actual_run_size - (actual_nregs * 2743 bin_info->reg_interval) - pad_size + bin_info->redzone_size; 2744 2745 assert(bin_info->reg0_offset - bin_info->redzone_size + (bin_info->nregs 2746 * bin_info->reg_interval) + pad_size == bin_info->run_size); 2747} 2748 2749static void 2750bin_info_init(void) 2751{ 2752 arena_bin_info_t *bin_info; 2753 2754#define BIN_INFO_INIT_bin_yes(index, size) \ 2755 bin_info = &arena_bin_info[index]; \ 2756 bin_info->reg_size = size; \ 2757 bin_info_run_size_calc(bin_info); \ 2758 bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); 2759#define BIN_INFO_INIT_bin_no(index, size) 2760#define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \ 2761 BIN_INFO_INIT_bin_##bin(index, (ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta)) 2762 SIZE_CLASSES 2763#undef BIN_INFO_INIT_bin_yes 2764#undef BIN_INFO_INIT_bin_no 2765#undef SC 2766} 2767 2768void 2769arena_boot(void) 2770{ 2771 size_t header_size; 2772 unsigned i; 2773 2774 /* 2775 * Compute the header size such that it is large enough to contain the 2776 * page map. The page map is biased to omit entries for the header 2777 * itself, so some iteration is necessary to compute the map bias. 2778 * 2779 * 1) Compute safe header_size and map_bias values that include enough 2780 * space for an unbiased page map. 2781 * 2) Refine map_bias based on (1) to omit the header pages in the page 2782 * map. The resulting map_bias may be one too small. 2783 * 3) Refine map_bias based on (2). The result will be >= the result 2784 * from (2), and will always be correct. 2785 */ 2786 map_bias = 0; 2787 for (i = 0; i < 3; i++) { 2788 header_size = offsetof(arena_chunk_t, map_bits) + 2789 ((sizeof(arena_chunk_map_bits_t) + 2790 sizeof(arena_chunk_map_misc_t)) * (chunk_npages-map_bias)); 2791 map_bias = (header_size + PAGE_MASK) >> LG_PAGE; 2792 } 2793 assert(map_bias > 0); 2794 2795 map_misc_offset = offsetof(arena_chunk_t, map_bits) + 2796 sizeof(arena_chunk_map_bits_t) * (chunk_npages-map_bias); 2797 2798 arena_maxrun = chunksize - (map_bias << LG_PAGE); 2799 assert(arena_maxrun > 0); 2800 arena_maxclass = index2size(size2index(chunksize)-1); 2801 if (arena_maxclass > arena_maxrun) { 2802 /* 2803 * For small chunk sizes it's possible for there to be fewer 2804 * non-header pages available than are necessary to serve the 2805 * size classes just below chunksize. 2806 */ 2807 arena_maxclass = arena_maxrun; 2808 } 2809 assert(arena_maxclass > 0); 2810 nlclasses = size2index(arena_maxclass) - size2index(SMALL_MAXCLASS); 2811 nhclasses = NSIZES - nlclasses - NBINS; 2812 2813 bin_info_init(); 2814} 2815 2816void 2817arena_prefork(arena_t *arena) 2818{ 2819 unsigned i; 2820 2821 malloc_mutex_prefork(&arena->lock); 2822 malloc_mutex_prefork(&arena->huge_mtx); 2823 malloc_mutex_prefork(&arena->chunks_mtx); 2824 malloc_mutex_prefork(&arena->node_cache_mtx); 2825 for (i = 0; i < NBINS; i++) 2826 malloc_mutex_prefork(&arena->bins[i].lock); 2827} 2828 2829void 2830arena_postfork_parent(arena_t *arena) 2831{ 2832 unsigned i; 2833 2834 for (i = 0; i < NBINS; i++) 2835 malloc_mutex_postfork_parent(&arena->bins[i].lock); 2836 malloc_mutex_postfork_parent(&arena->node_cache_mtx); 2837 malloc_mutex_postfork_parent(&arena->chunks_mtx); 2838 malloc_mutex_postfork_parent(&arena->huge_mtx); 2839 malloc_mutex_postfork_parent(&arena->lock); 2840} 2841 2842void 2843arena_postfork_child(arena_t *arena) 2844{ 2845 unsigned i; 2846 2847 for (i = 0; i < NBINS; i++) 2848 malloc_mutex_postfork_child(&arena->bins[i].lock); 2849 malloc_mutex_postfork_child(&arena->node_cache_mtx); 2850 malloc_mutex_postfork_child(&arena->chunks_mtx); 2851 malloc_mutex_postfork_child(&arena->huge_mtx); 2852 malloc_mutex_postfork_child(&arena->lock); 2853} 2854