1#define JEMALLOC_C_ 2#include "jemalloc/internal/jemalloc_internal.h" 3 4/******************************************************************************/ 5/* Data. */ 6 7/* Runtime configuration options. */ 8const char *je_malloc_conf 9#ifndef _WIN32 10 JEMALLOC_ATTR(weak) 11#endif 12 ; 13bool opt_abort = 14#ifdef JEMALLOC_DEBUG 15 true 16#else 17 false 18#endif 19 ; 20const char *opt_junk = 21#if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) 22 "true" 23#else 24 "false" 25#endif 26 ; 27bool opt_junk_alloc = 28#if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) 29 true 30#else 31 false 32#endif 33 ; 34bool opt_junk_free = 35#if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) 36 true 37#else 38 false 39#endif 40 ; 41 42size_t opt_quarantine = ZU(0); 43bool opt_redzone = false; 44bool opt_utrace = false; 45bool opt_xmalloc = false; 46bool opt_zero = false; 47unsigned opt_narenas = 0; 48 49/* Initialized to true if the process is running inside Valgrind. */ 50bool in_valgrind; 51 52unsigned ncpus; 53 54/* Protects arenas initialization. */ 55static malloc_mutex_t arenas_lock; 56/* 57 * Arenas that are used to service external requests. Not all elements of the 58 * arenas array are necessarily used; arenas are created lazily as needed. 59 * 60 * arenas[0..narenas_auto) are used for automatic multiplexing of threads and 61 * arenas. arenas[narenas_auto..narenas_total) are only used if the application 62 * takes some action to create them and allocate from them. 63 */ 64arena_t **arenas; 65static unsigned narenas_total; /* Use narenas_total_*(). */ 66static arena_t *a0; /* arenas[0]; read-only after initialization. */ 67unsigned narenas_auto; /* Read-only after initialization. */ 68 69typedef enum { 70 malloc_init_uninitialized = 3, 71 malloc_init_a0_initialized = 2, 72 malloc_init_recursible = 1, 73 malloc_init_initialized = 0 /* Common case --> jnz. */ 74} malloc_init_t; 75static malloc_init_t malloc_init_state = malloc_init_uninitialized; 76 77/* False should be the common case. Set to true to trigger initialization. */ 78static bool malloc_slow = true; 79 80/* When malloc_slow is true, set the corresponding bits for sanity check. */ 81enum { 82 flag_opt_junk_alloc = (1U), 83 flag_opt_junk_free = (1U << 1), 84 flag_opt_quarantine = (1U << 2), 85 flag_opt_zero = (1U << 3), 86 flag_opt_utrace = (1U << 4), 87 flag_in_valgrind = (1U << 5), 88 flag_opt_xmalloc = (1U << 6) 89}; 90static uint8_t malloc_slow_flags; 91 92JEMALLOC_ALIGNED(CACHELINE) 93const size_t pind2sz_tab[NPSIZES] = { 94#define PSZ_yes(lg_grp, ndelta, lg_delta) \ 95 (((ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta))), 96#define PSZ_no(lg_grp, ndelta, lg_delta) 97#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup) \ 98 PSZ_##psz(lg_grp, ndelta, lg_delta) 99 SIZE_CLASSES 100#undef PSZ_yes 101#undef PSZ_no 102#undef SC 103}; 104 105JEMALLOC_ALIGNED(CACHELINE) 106const size_t index2size_tab[NSIZES] = { 107#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup) \ 108 ((ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta)), 109 SIZE_CLASSES 110#undef SC 111}; 112 113JEMALLOC_ALIGNED(CACHELINE) 114const uint8_t size2index_tab[] = { 115#if LG_TINY_MIN == 0 116#warning "Dangerous LG_TINY_MIN" 117#define S2B_0(i) i, 118#elif LG_TINY_MIN == 1 119#warning "Dangerous LG_TINY_MIN" 120#define S2B_1(i) i, 121#elif LG_TINY_MIN == 2 122#warning "Dangerous LG_TINY_MIN" 123#define S2B_2(i) i, 124#elif LG_TINY_MIN == 3 125#define S2B_3(i) i, 126#elif LG_TINY_MIN == 4 127#define S2B_4(i) i, 128#elif LG_TINY_MIN == 5 129#define S2B_5(i) i, 130#elif LG_TINY_MIN == 6 131#define S2B_6(i) i, 132#elif LG_TINY_MIN == 7 133#define S2B_7(i) i, 134#elif LG_TINY_MIN == 8 135#define S2B_8(i) i, 136#elif LG_TINY_MIN == 9 137#define S2B_9(i) i, 138#elif LG_TINY_MIN == 10 139#define S2B_10(i) i, 140#elif LG_TINY_MIN == 11 141#define S2B_11(i) i, 142#else 143#error "Unsupported LG_TINY_MIN" 144#endif 145#if LG_TINY_MIN < 1 146#define S2B_1(i) S2B_0(i) S2B_0(i) 147#endif 148#if LG_TINY_MIN < 2 149#define S2B_2(i) S2B_1(i) S2B_1(i) 150#endif 151#if LG_TINY_MIN < 3 152#define S2B_3(i) S2B_2(i) S2B_2(i) 153#endif 154#if LG_TINY_MIN < 4 155#define S2B_4(i) S2B_3(i) S2B_3(i) 156#endif 157#if LG_TINY_MIN < 5 158#define S2B_5(i) S2B_4(i) S2B_4(i) 159#endif 160#if LG_TINY_MIN < 6 161#define S2B_6(i) S2B_5(i) S2B_5(i) 162#endif 163#if LG_TINY_MIN < 7 164#define S2B_7(i) S2B_6(i) S2B_6(i) 165#endif 166#if LG_TINY_MIN < 8 167#define S2B_8(i) S2B_7(i) S2B_7(i) 168#endif 169#if LG_TINY_MIN < 9 170#define S2B_9(i) S2B_8(i) S2B_8(i) 171#endif 172#if LG_TINY_MIN < 10 173#define S2B_10(i) S2B_9(i) S2B_9(i) 174#endif 175#if LG_TINY_MIN < 11 176#define S2B_11(i) S2B_10(i) S2B_10(i) 177#endif 178#define S2B_no(i) 179#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup) \ 180 S2B_##lg_delta_lookup(index) 181 SIZE_CLASSES 182#undef S2B_3 183#undef S2B_4 184#undef S2B_5 185#undef S2B_6 186#undef S2B_7 187#undef S2B_8 188#undef S2B_9 189#undef S2B_10 190#undef S2B_11 191#undef S2B_no 192#undef SC 193}; 194 195#ifdef JEMALLOC_THREADED_INIT 196/* Used to let the initializing thread recursively allocate. */ 197# define NO_INITIALIZER ((unsigned long)0) 198# define INITIALIZER pthread_self() 199# define IS_INITIALIZER (malloc_initializer == pthread_self()) 200static pthread_t malloc_initializer = NO_INITIALIZER; 201#else 202# define NO_INITIALIZER false 203# define INITIALIZER true 204# define IS_INITIALIZER malloc_initializer 205static bool malloc_initializer = NO_INITIALIZER; 206#endif 207 208/* Used to avoid initialization races. */ 209#ifdef _WIN32 210#if _WIN32_WINNT >= 0x0600 211static malloc_mutex_t init_lock = SRWLOCK_INIT; 212#else 213static malloc_mutex_t init_lock; 214static bool init_lock_initialized = false; 215 216JEMALLOC_ATTR(constructor) 217static void WINAPI 218_init_init_lock(void) 219{ 220 221 /* If another constructor in the same binary is using mallctl to 222 * e.g. setup chunk hooks, it may end up running before this one, 223 * and malloc_init_hard will crash trying to lock the uninitialized 224 * lock. So we force an initialization of the lock in 225 * malloc_init_hard as well. We don't try to care about atomicity 226 * of the accessed to the init_lock_initialized boolean, since it 227 * really only matters early in the process creation, before any 228 * separate thread normally starts doing anything. */ 229 if (!init_lock_initialized) 230 malloc_mutex_init(&init_lock, "init", WITNESS_RANK_INIT); 231 init_lock_initialized = true; 232} 233 234#ifdef _MSC_VER 235# pragma section(".CRT$XCU", read) 236JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used) 237static const void (WINAPI *init_init_lock)(void) = _init_init_lock; 238#endif 239#endif 240#else 241static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER; 242#endif 243 244typedef struct { 245 void *p; /* Input pointer (as in realloc(p, s)). */ 246 size_t s; /* Request size. */ 247 void *r; /* Result pointer. */ 248} malloc_utrace_t; 249 250#ifdef JEMALLOC_UTRACE 251# define UTRACE(a, b, c) do { \ 252 if (unlikely(opt_utrace)) { \ 253 int utrace_serrno = errno; \ 254 malloc_utrace_t ut; \ 255 ut.p = (a); \ 256 ut.s = (b); \ 257 ut.r = (c); \ 258 utrace(&ut, sizeof(ut)); \ 259 errno = utrace_serrno; \ 260 } \ 261} while (0) 262#else 263# define UTRACE(a, b, c) 264#endif 265 266/******************************************************************************/ 267/* 268 * Function prototypes for static functions that are referenced prior to 269 * definition. 270 */ 271 272static bool malloc_init_hard_a0(void); 273static bool malloc_init_hard(void); 274 275/******************************************************************************/ 276/* 277 * Begin miscellaneous support functions. 278 */ 279 280JEMALLOC_ALWAYS_INLINE_C bool 281malloc_initialized(void) 282{ 283 284 return (malloc_init_state == malloc_init_initialized); 285} 286 287JEMALLOC_ALWAYS_INLINE_C void 288malloc_thread_init(void) 289{ 290 291 /* 292 * TSD initialization can't be safely done as a side effect of 293 * deallocation, because it is possible for a thread to do nothing but 294 * deallocate its TLS data via free(), in which case writing to TLS 295 * would cause write-after-free memory corruption. The quarantine 296 * facility *only* gets used as a side effect of deallocation, so make 297 * a best effort attempt at initializing its TSD by hooking all 298 * allocation events. 299 */ 300 if (config_fill && unlikely(opt_quarantine)) 301 quarantine_alloc_hook(); 302} 303 304JEMALLOC_ALWAYS_INLINE_C bool 305malloc_init_a0(void) 306{ 307 308 if (unlikely(malloc_init_state == malloc_init_uninitialized)) 309 return (malloc_init_hard_a0()); 310 return (false); 311} 312 313JEMALLOC_ALWAYS_INLINE_C bool 314malloc_init(void) 315{ 316 317 if (unlikely(!malloc_initialized()) && malloc_init_hard()) 318 return (true); 319 malloc_thread_init(); 320 321 return (false); 322} 323 324/* 325 * The a0*() functions are used instead of i{d,}alloc() in situations that 326 * cannot tolerate TLS variable access. 327 */ 328 329static void * 330a0ialloc(size_t size, bool zero, bool is_metadata) 331{ 332 333 if (unlikely(malloc_init_a0())) 334 return (NULL); 335 336 return (iallocztm(TSDN_NULL, size, size2index(size), zero, NULL, 337 is_metadata, arena_get(TSDN_NULL, 0, true), true)); 338} 339 340static void 341a0idalloc(void *ptr, bool is_metadata) 342{ 343 344 idalloctm(TSDN_NULL, ptr, false, is_metadata, true); 345} 346 347arena_t * 348a0get(void) 349{ 350 351 return (a0); 352} 353 354void * 355a0malloc(size_t size) 356{ 357 358 return (a0ialloc(size, false, true)); 359} 360 361void 362a0dalloc(void *ptr) 363{ 364 365 a0idalloc(ptr, true); 366} 367 368/* 369 * FreeBSD's libc uses the bootstrap_*() functions in bootstrap-senstive 370 * situations that cannot tolerate TLS variable access (TLS allocation and very 371 * early internal data structure initialization). 372 */ 373 374void * 375bootstrap_malloc(size_t size) 376{ 377 378 if (unlikely(size == 0)) 379 size = 1; 380 381 return (a0ialloc(size, false, false)); 382} 383 384void * 385bootstrap_calloc(size_t num, size_t size) 386{ 387 size_t num_size; 388 389 num_size = num * size; 390 if (unlikely(num_size == 0)) { 391 assert(num == 0 || size == 0); 392 num_size = 1; 393 } 394 395 return (a0ialloc(num_size, true, false)); 396} 397 398void 399bootstrap_free(void *ptr) 400{ 401 402 if (unlikely(ptr == NULL)) 403 return; 404 405 a0idalloc(ptr, false); 406} 407 408static void 409arena_set(unsigned ind, arena_t *arena) 410{ 411 412 atomic_write_p((void **)&arenas[ind], arena); 413} 414 415static void 416narenas_total_set(unsigned narenas) 417{ 418 419 atomic_write_u(&narenas_total, narenas); 420} 421 422static void 423narenas_total_inc(void) 424{ 425 426 atomic_add_u(&narenas_total, 1); 427} 428 429unsigned 430narenas_total_get(void) 431{ 432 433 return (atomic_read_u(&narenas_total)); 434} 435 436/* Create a new arena and insert it into the arenas array at index ind. */ 437static arena_t * 438arena_init_locked(tsdn_t *tsdn, unsigned ind) 439{ 440 arena_t *arena; 441 442 assert(ind <= narenas_total_get()); 443 if (ind > MALLOCX_ARENA_MAX) 444 return (NULL); 445 if (ind == narenas_total_get()) 446 narenas_total_inc(); 447 448 /* 449 * Another thread may have already initialized arenas[ind] if it's an 450 * auto arena. 451 */ 452 arena = arena_get(tsdn, ind, false); 453 if (arena != NULL) { 454 assert(ind < narenas_auto); 455 return (arena); 456 } 457 458 /* Actually initialize the arena. */ 459 arena = arena_new(tsdn, ind); 460 arena_set(ind, arena); 461 return (arena); 462} 463 464arena_t * 465arena_init(tsdn_t *tsdn, unsigned ind) 466{ 467 arena_t *arena; 468 469 malloc_mutex_lock(tsdn, &arenas_lock); 470 arena = arena_init_locked(tsdn, ind); 471 malloc_mutex_unlock(tsdn, &arenas_lock); 472 return (arena); 473} 474 475static void 476arena_bind(tsd_t *tsd, unsigned ind, bool internal) 477{ 478 arena_t *arena; 479 480 if (!tsd_nominal(tsd)) 481 return; 482 483 arena = arena_get(tsd_tsdn(tsd), ind, false); 484 arena_nthreads_inc(arena, internal); 485 486 if (internal) 487 tsd_iarena_set(tsd, arena); 488 else 489 tsd_arena_set(tsd, arena); 490} 491 492void 493arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind) 494{ 495 arena_t *oldarena, *newarena; 496 497 oldarena = arena_get(tsd_tsdn(tsd), oldind, false); 498 newarena = arena_get(tsd_tsdn(tsd), newind, false); 499 arena_nthreads_dec(oldarena, false); 500 arena_nthreads_inc(newarena, false); 501 tsd_arena_set(tsd, newarena); 502} 503 504static void 505arena_unbind(tsd_t *tsd, unsigned ind, bool internal) 506{ 507 arena_t *arena; 508 509 arena = arena_get(tsd_tsdn(tsd), ind, false); 510 arena_nthreads_dec(arena, internal); 511 if (internal) 512 tsd_iarena_set(tsd, NULL); 513 else 514 tsd_arena_set(tsd, NULL); 515} 516 517arena_tdata_t * 518arena_tdata_get_hard(tsd_t *tsd, unsigned ind) 519{ 520 arena_tdata_t *tdata, *arenas_tdata_old; 521 arena_tdata_t *arenas_tdata = tsd_arenas_tdata_get(tsd); 522 unsigned narenas_tdata_old, i; 523 unsigned narenas_tdata = tsd_narenas_tdata_get(tsd); 524 unsigned narenas_actual = narenas_total_get(); 525 526 /* 527 * Dissociate old tdata array (and set up for deallocation upon return) 528 * if it's too small. 529 */ 530 if (arenas_tdata != NULL && narenas_tdata < narenas_actual) { 531 arenas_tdata_old = arenas_tdata; 532 narenas_tdata_old = narenas_tdata; 533 arenas_tdata = NULL; 534 narenas_tdata = 0; 535 tsd_arenas_tdata_set(tsd, arenas_tdata); 536 tsd_narenas_tdata_set(tsd, narenas_tdata); 537 } else { 538 arenas_tdata_old = NULL; 539 narenas_tdata_old = 0; 540 } 541 542 /* Allocate tdata array if it's missing. */ 543 if (arenas_tdata == NULL) { 544 bool *arenas_tdata_bypassp = tsd_arenas_tdata_bypassp_get(tsd); 545 narenas_tdata = (ind < narenas_actual) ? narenas_actual : ind+1; 546 547 if (tsd_nominal(tsd) && !*arenas_tdata_bypassp) { 548 *arenas_tdata_bypassp = true; 549 arenas_tdata = (arena_tdata_t *)a0malloc( 550 sizeof(arena_tdata_t) * narenas_tdata); 551 *arenas_tdata_bypassp = false; 552 } 553 if (arenas_tdata == NULL) { 554 tdata = NULL; 555 goto label_return; 556 } 557 assert(tsd_nominal(tsd) && !*arenas_tdata_bypassp); 558 tsd_arenas_tdata_set(tsd, arenas_tdata); 559 tsd_narenas_tdata_set(tsd, narenas_tdata); 560 } 561 562 /* 563 * Copy to tdata array. It's possible that the actual number of arenas 564 * has increased since narenas_total_get() was called above, but that 565 * causes no correctness issues unless two threads concurrently execute 566 * the arenas.extend mallctl, which we trust mallctl synchronization to 567 * prevent. 568 */ 569 570 /* Copy/initialize tickers. */ 571 for (i = 0; i < narenas_actual; i++) { 572 if (i < narenas_tdata_old) { 573 ticker_copy(&arenas_tdata[i].decay_ticker, 574 &arenas_tdata_old[i].decay_ticker); 575 } else { 576 ticker_init(&arenas_tdata[i].decay_ticker, 577 DECAY_NTICKS_PER_UPDATE); 578 } 579 } 580 if (narenas_tdata > narenas_actual) { 581 memset(&arenas_tdata[narenas_actual], 0, sizeof(arena_tdata_t) 582 * (narenas_tdata - narenas_actual)); 583 } 584 585 /* Read the refreshed tdata array. */ 586 tdata = &arenas_tdata[ind]; 587label_return: 588 if (arenas_tdata_old != NULL) 589 a0dalloc(arenas_tdata_old); 590 return (tdata); 591} 592 593/* Slow path, called only by arena_choose(). */ 594arena_t * 595arena_choose_hard(tsd_t *tsd, bool internal) 596{ 597 arena_t *ret JEMALLOC_CC_SILENCE_INIT(NULL); 598 599 if (narenas_auto > 1) { 600 unsigned i, j, choose[2], first_null; 601 602 /* 603 * Determine binding for both non-internal and internal 604 * allocation. 605 * 606 * choose[0]: For application allocation. 607 * choose[1]: For internal metadata allocation. 608 */ 609 610 for (j = 0; j < 2; j++) 611 choose[j] = 0; 612 613 first_null = narenas_auto; 614 malloc_mutex_lock(tsd_tsdn(tsd), &arenas_lock); 615 assert(arena_get(tsd_tsdn(tsd), 0, false) != NULL); 616 for (i = 1; i < narenas_auto; i++) { 617 if (arena_get(tsd_tsdn(tsd), i, false) != NULL) { 618 /* 619 * Choose the first arena that has the lowest 620 * number of threads assigned to it. 621 */ 622 for (j = 0; j < 2; j++) { 623 if (arena_nthreads_get(arena_get( 624 tsd_tsdn(tsd), i, false), !!j) < 625 arena_nthreads_get(arena_get( 626 tsd_tsdn(tsd), choose[j], false), 627 !!j)) 628 choose[j] = i; 629 } 630 } else if (first_null == narenas_auto) { 631 /* 632 * Record the index of the first uninitialized 633 * arena, in case all extant arenas are in use. 634 * 635 * NB: It is possible for there to be 636 * discontinuities in terms of initialized 637 * versus uninitialized arenas, due to the 638 * "thread.arena" mallctl. 639 */ 640 first_null = i; 641 } 642 } 643 644 for (j = 0; j < 2; j++) { 645 if (arena_nthreads_get(arena_get(tsd_tsdn(tsd), 646 choose[j], false), !!j) == 0 || first_null == 647 narenas_auto) { 648 /* 649 * Use an unloaded arena, or the least loaded 650 * arena if all arenas are already initialized. 651 */ 652 if (!!j == internal) { 653 ret = arena_get(tsd_tsdn(tsd), 654 choose[j], false); 655 } 656 } else { 657 arena_t *arena; 658 659 /* Initialize a new arena. */ 660 choose[j] = first_null; 661 arena = arena_init_locked(tsd_tsdn(tsd), 662 choose[j]); 663 if (arena == NULL) { 664 malloc_mutex_unlock(tsd_tsdn(tsd), 665 &arenas_lock); 666 return (NULL); 667 } 668 if (!!j == internal) 669 ret = arena; 670 } 671 arena_bind(tsd, choose[j], !!j); 672 } 673 malloc_mutex_unlock(tsd_tsdn(tsd), &arenas_lock); 674 } else { 675 ret = arena_get(tsd_tsdn(tsd), 0, false); 676 arena_bind(tsd, 0, false); 677 arena_bind(tsd, 0, true); 678 } 679 680 return (ret); 681} 682 683void 684thread_allocated_cleanup(tsd_t *tsd) 685{ 686 687 /* Do nothing. */ 688} 689 690void 691thread_deallocated_cleanup(tsd_t *tsd) 692{ 693 694 /* Do nothing. */ 695} 696 697void 698iarena_cleanup(tsd_t *tsd) 699{ 700 arena_t *iarena; 701 702 iarena = tsd_iarena_get(tsd); 703 if (iarena != NULL) 704 arena_unbind(tsd, iarena->ind, true); 705} 706 707void 708arena_cleanup(tsd_t *tsd) 709{ 710 arena_t *arena; 711 712 arena = tsd_arena_get(tsd); 713 if (arena != NULL) 714 arena_unbind(tsd, arena->ind, false); 715} 716 717void 718arenas_tdata_cleanup(tsd_t *tsd) 719{ 720 arena_tdata_t *arenas_tdata; 721 722 /* Prevent tsd->arenas_tdata from being (re)created. */ 723 *tsd_arenas_tdata_bypassp_get(tsd) = true; 724 725 arenas_tdata = tsd_arenas_tdata_get(tsd); 726 if (arenas_tdata != NULL) { 727 tsd_arenas_tdata_set(tsd, NULL); 728 a0dalloc(arenas_tdata); 729 } 730} 731 732void 733narenas_tdata_cleanup(tsd_t *tsd) 734{ 735 736 /* Do nothing. */ 737} 738 739void 740arenas_tdata_bypass_cleanup(tsd_t *tsd) 741{ 742 743 /* Do nothing. */ 744} 745 746static void 747stats_print_atexit(void) 748{ 749 750 if (config_tcache && config_stats) { 751 tsdn_t *tsdn; 752 unsigned narenas, i; 753 754 tsdn = tsdn_fetch(); 755 756 /* 757 * Merge stats from extant threads. This is racy, since 758 * individual threads do not lock when recording tcache stats 759 * events. As a consequence, the final stats may be slightly 760 * out of date by the time they are reported, if other threads 761 * continue to allocate. 762 */ 763 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { 764 arena_t *arena = arena_get(tsdn, i, false); 765 if (arena != NULL) { 766 tcache_t *tcache; 767 768 /* 769 * tcache_stats_merge() locks bins, so if any 770 * code is introduced that acquires both arena 771 * and bin locks in the opposite order, 772 * deadlocks may result. 773 */ 774 malloc_mutex_lock(tsdn, &arena->lock); 775 ql_foreach(tcache, &arena->tcache_ql, link) { 776 tcache_stats_merge(tsdn, tcache, arena); 777 } 778 malloc_mutex_unlock(tsdn, &arena->lock); 779 } 780 } 781 } 782 je_malloc_stats_print(NULL, NULL, NULL); 783} 784 785/* 786 * End miscellaneous support functions. 787 */ 788/******************************************************************************/ 789/* 790 * Begin initialization functions. 791 */ 792 793#ifndef JEMALLOC_HAVE_SECURE_GETENV 794static char * 795secure_getenv(const char *name) 796{ 797 798# ifdef JEMALLOC_HAVE_ISSETUGID 799 if (issetugid() != 0) 800 return (NULL); 801# endif 802 return (getenv(name)); 803} 804#endif 805 806static unsigned 807malloc_ncpus(void) 808{ 809 long result; 810 811#ifdef _WIN32 812 SYSTEM_INFO si; 813 GetSystemInfo(&si); 814 result = si.dwNumberOfProcessors; 815#elif defined(JEMALLOC_GLIBC_MALLOC_HOOK) && defined(CPU_COUNT) 816 /* 817 * glibc >= 2.6 has the CPU_COUNT macro. 818 * 819 * glibc's sysconf() uses isspace(). glibc allocates for the first time 820 * *before* setting up the isspace tables. Therefore we need a 821 * different method to get the number of CPUs. 822 */ 823 { 824 cpu_set_t set; 825 826 pthread_getaffinity_np(pthread_self(), sizeof(set), &set); 827 result = CPU_COUNT(&set); 828 } 829#else 830 result = sysconf(_SC_NPROCESSORS_ONLN); 831#endif 832 return ((result == -1) ? 1 : (unsigned)result); 833} 834 835static bool 836malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, 837 char const **v_p, size_t *vlen_p) 838{ 839 bool accept; 840 const char *opts = *opts_p; 841 842 *k_p = opts; 843 844 for (accept = false; !accept;) { 845 switch (*opts) { 846 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': 847 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': 848 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': 849 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': 850 case 'Y': case 'Z': 851 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': 852 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': 853 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': 854 case 's': case 't': case 'u': case 'v': case 'w': case 'x': 855 case 'y': case 'z': 856 case '0': case '1': case '2': case '3': case '4': case '5': 857 case '6': case '7': case '8': case '9': 858 case '_': 859 opts++; 860 break; 861 case ':': 862 opts++; 863 *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p; 864 *v_p = opts; 865 accept = true; 866 break; 867 case '\0': 868 if (opts != *opts_p) { 869 malloc_write("<jemalloc>: Conf string ends " 870 "with key\n"); 871 } 872 return (true); 873 default: 874 malloc_write("<jemalloc>: Malformed conf string\n"); 875 return (true); 876 } 877 } 878 879 for (accept = false; !accept;) { 880 switch (*opts) { 881 case ',': 882 opts++; 883 /* 884 * Look ahead one character here, because the next time 885 * this function is called, it will assume that end of 886 * input has been cleanly reached if no input remains, 887 * but we have optimistically already consumed the 888 * comma if one exists. 889 */ 890 if (*opts == '\0') { 891 malloc_write("<jemalloc>: Conf string ends " 892 "with comma\n"); 893 } 894 *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p; 895 accept = true; 896 break; 897 case '\0': 898 *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p; 899 accept = true; 900 break; 901 default: 902 opts++; 903 break; 904 } 905 } 906 907 *opts_p = opts; 908 return (false); 909} 910 911static void 912malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, 913 size_t vlen) 914{ 915 916 malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k, 917 (int)vlen, v); 918} 919 920static void 921malloc_slow_flag_init(void) 922{ 923 /* 924 * Combine the runtime options into malloc_slow for fast path. Called 925 * after processing all the options. 926 */ 927 malloc_slow_flags |= (opt_junk_alloc ? flag_opt_junk_alloc : 0) 928 | (opt_junk_free ? flag_opt_junk_free : 0) 929 | (opt_quarantine ? flag_opt_quarantine : 0) 930 | (opt_zero ? flag_opt_zero : 0) 931 | (opt_utrace ? flag_opt_utrace : 0) 932 | (opt_xmalloc ? flag_opt_xmalloc : 0); 933 934 if (config_valgrind) 935 malloc_slow_flags |= (in_valgrind ? flag_in_valgrind : 0); 936 937 malloc_slow = (malloc_slow_flags != 0); 938} 939 940static void 941malloc_conf_init(void) 942{ 943 unsigned i; 944 char buf[PATH_MAX + 1]; 945 const char *opts, *k, *v; 946 size_t klen, vlen; 947 948 /* 949 * Automatically configure valgrind before processing options. The 950 * valgrind option remains in jemalloc 3.x for compatibility reasons. 951 */ 952 if (config_valgrind) { 953 in_valgrind = (RUNNING_ON_VALGRIND != 0) ? true : false; 954 if (config_fill && unlikely(in_valgrind)) { 955 opt_junk = "false"; 956 opt_junk_alloc = false; 957 opt_junk_free = false; 958 assert(!opt_zero); 959 opt_quarantine = JEMALLOC_VALGRIND_QUARANTINE_DEFAULT; 960 opt_redzone = true; 961 } 962 if (config_tcache && unlikely(in_valgrind)) 963 opt_tcache = false; 964 } 965 966#if defined(__ANDROID__) 967 for (i = 0; i < 2; i++) { 968#else 969 for (i = 0; i < 4; i++) { 970#endif 971 /* Get runtime configuration. */ 972 switch (i) { 973 case 0: 974 opts = config_malloc_conf; 975 break; 976 case 1: 977 if (je_malloc_conf != NULL) { 978 /* 979 * Use options that were compiled into the 980 * program. 981 */ 982 opts = je_malloc_conf; 983 } else { 984 /* No configuration specified. */ 985 buf[0] = '\0'; 986 opts = buf; 987 } 988 break; 989 case 2: { 990 ssize_t linklen = 0; 991#ifndef _WIN32 992 int saved_errno = errno; 993 const char *linkname = 994# ifdef JEMALLOC_PREFIX 995 "/etc/"JEMALLOC_PREFIX"malloc.conf" 996# else 997 "/etc/malloc.conf" 998# endif 999 ; 1000 1001 /* 1002 * Try to use the contents of the "/etc/malloc.conf" 1003 * symbolic link's name. 1004 */ 1005 linklen = readlink(linkname, buf, sizeof(buf) - 1); 1006 if (linklen == -1) { 1007 /* No configuration specified. */ 1008 linklen = 0; 1009 /* Restore errno. */ 1010 set_errno(saved_errno); 1011 } 1012#endif 1013 buf[linklen] = '\0'; 1014 opts = buf; 1015 break; 1016 } case 3: { 1017 const char *envname = 1018#ifdef JEMALLOC_PREFIX 1019 JEMALLOC_CPREFIX"MALLOC_CONF" 1020#else 1021 "MALLOC_CONF" 1022#endif 1023 ; 1024 1025 if ((opts = secure_getenv(envname)) != NULL) { 1026 /* 1027 * Do nothing; opts is already initialized to 1028 * the value of the MALLOC_CONF environment 1029 * variable. 1030 */ 1031 } else { 1032 /* No configuration specified. */ 1033 buf[0] = '\0'; 1034 opts = buf; 1035 } 1036 break; 1037 } default: 1038 not_reached(); 1039 buf[0] = '\0'; 1040 opts = buf; 1041 } 1042 1043 while (*opts != '\0' && !malloc_conf_next(&opts, &k, &klen, &v, 1044 &vlen)) { 1045#define CONF_MATCH(n) \ 1046 (sizeof(n)-1 == klen && strncmp(n, k, klen) == 0) 1047#define CONF_MATCH_VALUE(n) \ 1048 (sizeof(n)-1 == vlen && strncmp(n, v, vlen) == 0) 1049#define CONF_HANDLE_BOOL(o, n, cont) \ 1050 if (CONF_MATCH(n)) { \ 1051 if (CONF_MATCH_VALUE("true")) \ 1052 o = true; \ 1053 else if (CONF_MATCH_VALUE("false")) \ 1054 o = false; \ 1055 else { \ 1056 malloc_conf_error( \ 1057 "Invalid conf value", \ 1058 k, klen, v, vlen); \ 1059 } \ 1060 if (cont) \ 1061 continue; \ 1062 } 1063#define CONF_MIN_no(um, min) false 1064#define CONF_MIN_yes(um, min) ((um) < (min)) 1065#define CONF_MAX_no(um, max) false 1066#define CONF_MAX_yes(um, max) ((um) > (max)) 1067#define CONF_HANDLE_T_U(t, o, n, min, max, check_min, check_max, clip) \ 1068 if (CONF_MATCH(n)) { \ 1069 uintmax_t um; \ 1070 char *end; \ 1071 \ 1072 set_errno(0); \ 1073 um = malloc_strtoumax(v, &end, 0); \ 1074 if (get_errno() != 0 || (uintptr_t)end -\ 1075 (uintptr_t)v != vlen) { \ 1076 malloc_conf_error( \ 1077 "Invalid conf value", \ 1078 k, klen, v, vlen); \ 1079 } else if (clip) { \ 1080 if (CONF_MIN_##check_min(um, \ 1081 (min))) \ 1082 o = (t)(min); \ 1083 else if (CONF_MAX_##check_max( \ 1084 um, (max))) \ 1085 o = (t)(max); \ 1086 else \ 1087 o = (t)um; \ 1088 } else { \ 1089 if (CONF_MIN_##check_min(um, \ 1090 (min)) || \ 1091 CONF_MAX_##check_max(um, \ 1092 (max))) { \ 1093 malloc_conf_error( \ 1094 "Out-of-range " \ 1095 "conf value", \ 1096 k, klen, v, vlen); \ 1097 } else \ 1098 o = (t)um; \ 1099 } \ 1100 continue; \ 1101 } 1102#define CONF_HANDLE_UNSIGNED(o, n, min, max, check_min, check_max, \ 1103 clip) \ 1104 CONF_HANDLE_T_U(unsigned, o, n, min, max, \ 1105 check_min, check_max, clip) 1106#define CONF_HANDLE_SIZE_T(o, n, min, max, check_min, check_max, clip) \ 1107 CONF_HANDLE_T_U(size_t, o, n, min, max, \ 1108 check_min, check_max, clip) 1109#define CONF_HANDLE_SSIZE_T(o, n, min, max) \ 1110 if (CONF_MATCH(n)) { \ 1111 long l; \ 1112 char *end; \ 1113 \ 1114 set_errno(0); \ 1115 l = strtol(v, &end, 0); \ 1116 if (get_errno() != 0 || (uintptr_t)end -\ 1117 (uintptr_t)v != vlen) { \ 1118 malloc_conf_error( \ 1119 "Invalid conf value", \ 1120 k, klen, v, vlen); \ 1121 } else if (l < (ssize_t)(min) || l > \ 1122 (ssize_t)(max)) { \ 1123 malloc_conf_error( \ 1124 "Out-of-range conf value", \ 1125 k, klen, v, vlen); \ 1126 } else \ 1127 o = l; \ 1128 continue; \ 1129 } 1130#define CONF_HANDLE_CHAR_P(o, n, d) \ 1131 if (CONF_MATCH(n)) { \ 1132 size_t cpylen = (vlen <= \ 1133 sizeof(o)-1) ? vlen : \ 1134 sizeof(o)-1; \ 1135 strncpy(o, v, cpylen); \ 1136 o[cpylen] = '\0'; \ 1137 continue; \ 1138 } 1139 1140 CONF_HANDLE_BOOL(opt_abort, "abort", true) 1141 /* 1142 * Chunks always require at least one header page, 1143 * as many as 2^(LG_SIZE_CLASS_GROUP+1) data pages, and 1144 * possibly an additional page in the presence of 1145 * redzones. In order to simplify options processing, 1146 * use a conservative bound that accommodates all these 1147 * constraints. 1148 */ 1149 CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE + 1150 LG_SIZE_CLASS_GROUP + (config_fill ? 2 : 1), 1151 (sizeof(size_t) << 3) - 1, yes, yes, true) 1152 if (strncmp("dss", k, klen) == 0) { 1153 int i; 1154 bool match = false; 1155 for (i = 0; i < dss_prec_limit; i++) { 1156 if (strncmp(dss_prec_names[i], v, vlen) 1157 == 0) { 1158 if (chunk_dss_prec_set(i)) { 1159 malloc_conf_error( 1160 "Error setting dss", 1161 k, klen, v, vlen); 1162 } else { 1163 opt_dss = 1164 dss_prec_names[i]; 1165 match = true; 1166 break; 1167 } 1168 } 1169 } 1170 if (!match) { 1171 malloc_conf_error("Invalid conf value", 1172 k, klen, v, vlen); 1173 } 1174 continue; 1175 } 1176 CONF_HANDLE_UNSIGNED(opt_narenas, "narenas", 1, 1177 UINT_MAX, yes, no, false) 1178 if (strncmp("purge", k, klen) == 0) { 1179 int i; 1180 bool match = false; 1181 for (i = 0; i < purge_mode_limit; i++) { 1182 if (strncmp(purge_mode_names[i], v, 1183 vlen) == 0) { 1184 opt_purge = (purge_mode_t)i; 1185 match = true; 1186 break; 1187 } 1188 } 1189 if (!match) { 1190 malloc_conf_error("Invalid conf value", 1191 k, klen, v, vlen); 1192 } 1193 continue; 1194 } 1195 CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult", 1196 -1, (sizeof(size_t) << 3) - 1) 1197 CONF_HANDLE_SSIZE_T(opt_decay_time, "decay_time", -1, 1198 NSTIME_SEC_MAX); 1199 CONF_HANDLE_BOOL(opt_stats_print, "stats_print", true) 1200 if (config_fill) { 1201 if (CONF_MATCH("junk")) { 1202 if (CONF_MATCH_VALUE("true")) { 1203 if (config_valgrind && 1204 unlikely(in_valgrind)) { 1205 malloc_conf_error( 1206 "Deallocation-time " 1207 "junk filling cannot " 1208 "be enabled while " 1209 "running inside " 1210 "Valgrind", k, klen, v, 1211 vlen); 1212 } else { 1213 opt_junk = "true"; 1214 opt_junk_alloc = true; 1215 opt_junk_free = true; 1216 } 1217 } else if (CONF_MATCH_VALUE("false")) { 1218 opt_junk = "false"; 1219 opt_junk_alloc = opt_junk_free = 1220 false; 1221 } else if (CONF_MATCH_VALUE("alloc")) { 1222 opt_junk = "alloc"; 1223 opt_junk_alloc = true; 1224 opt_junk_free = false; 1225 } else if (CONF_MATCH_VALUE("free")) { 1226 if (config_valgrind && 1227 unlikely(in_valgrind)) { 1228 malloc_conf_error( 1229 "Deallocation-time " 1230 "junk filling cannot " 1231 "be enabled while " 1232 "running inside " 1233 "Valgrind", k, klen, v, 1234 vlen); 1235 } else { 1236 opt_junk = "free"; 1237 opt_junk_alloc = false; 1238 opt_junk_free = true; 1239 } 1240 } else { 1241 malloc_conf_error( 1242 "Invalid conf value", k, 1243 klen, v, vlen); 1244 } 1245 continue; 1246 } 1247 CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine", 1248 0, SIZE_T_MAX, no, no, false) 1249 CONF_HANDLE_BOOL(opt_redzone, "redzone", true) 1250 CONF_HANDLE_BOOL(opt_zero, "zero", true) 1251 } 1252 if (config_utrace) { 1253 CONF_HANDLE_BOOL(opt_utrace, "utrace", true) 1254 } 1255 if (config_xmalloc) { 1256 CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc", true) 1257 } 1258 if (config_tcache) { 1259 CONF_HANDLE_BOOL(opt_tcache, "tcache", 1260 !config_valgrind || !in_valgrind) 1261 if (CONF_MATCH("tcache")) { 1262 assert(config_valgrind && in_valgrind); 1263 if (opt_tcache) { 1264 opt_tcache = false; 1265 malloc_conf_error( 1266 "tcache cannot be enabled " 1267 "while running inside Valgrind", 1268 k, klen, v, vlen); 1269 } 1270 continue; 1271 } 1272 CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, 1273 "lg_tcache_max", -1, 1274 (sizeof(size_t) << 3) - 1) 1275 } 1276 if (config_prof) { 1277 CONF_HANDLE_BOOL(opt_prof, "prof", true) 1278 CONF_HANDLE_CHAR_P(opt_prof_prefix, 1279 "prof_prefix", "jeprof") 1280 CONF_HANDLE_BOOL(opt_prof_active, "prof_active", 1281 true) 1282 CONF_HANDLE_BOOL(opt_prof_thread_active_init, 1283 "prof_thread_active_init", true) 1284 CONF_HANDLE_SIZE_T(opt_lg_prof_sample, 1285 "lg_prof_sample", 0, (sizeof(uint64_t) << 3) 1286 - 1, no, yes, true) 1287 CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum", 1288 true) 1289 CONF_HANDLE_SSIZE_T(opt_lg_prof_interval, 1290 "lg_prof_interval", -1, 1291 (sizeof(uint64_t) << 3) - 1) 1292 CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump", 1293 true) 1294 CONF_HANDLE_BOOL(opt_prof_final, "prof_final", 1295 true) 1296 CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak", 1297 true) 1298 } 1299 malloc_conf_error("Invalid conf pair", k, klen, v, 1300 vlen); 1301#undef CONF_MATCH 1302#undef CONF_MATCH_VALUE 1303#undef CONF_HANDLE_BOOL 1304#undef CONF_MIN_no 1305#undef CONF_MIN_yes 1306#undef CONF_MAX_no 1307#undef CONF_MAX_yes 1308#undef CONF_HANDLE_T_U 1309#undef CONF_HANDLE_UNSIGNED 1310#undef CONF_HANDLE_SIZE_T 1311#undef CONF_HANDLE_SSIZE_T 1312#undef CONF_HANDLE_CHAR_P 1313 } 1314 } 1315} 1316 1317static bool 1318malloc_init_hard_needed(void) 1319{ 1320 1321 if (malloc_initialized() || (IS_INITIALIZER && malloc_init_state == 1322 malloc_init_recursible)) { 1323 /* 1324 * Another thread initialized the allocator before this one 1325 * acquired init_lock, or this thread is the initializing 1326 * thread, and it is recursively allocating. 1327 */ 1328 return (false); 1329 } 1330#ifdef JEMALLOC_THREADED_INIT 1331 if (malloc_initializer != NO_INITIALIZER && !IS_INITIALIZER) { 1332 spin_t spinner; 1333 1334 /* Busy-wait until the initializing thread completes. */ 1335 spin_init(&spinner); 1336 do { 1337 malloc_mutex_unlock(TSDN_NULL, &init_lock); 1338 spin_adaptive(&spinner); 1339 malloc_mutex_lock(TSDN_NULL, &init_lock); 1340 } while (!malloc_initialized()); 1341 return (false); 1342 } 1343#endif 1344 return (true); 1345} 1346 1347static bool 1348malloc_init_hard_a0_locked() 1349{ 1350 1351 malloc_initializer = INITIALIZER; 1352 1353 if (config_prof) 1354 prof_boot0(); 1355 malloc_conf_init(); 1356 if (opt_stats_print) { 1357 /* Print statistics at exit. */ 1358 if (atexit(stats_print_atexit) != 0) { 1359 malloc_write("<jemalloc>: Error in atexit()\n"); 1360 if (opt_abort) 1361 abort(); 1362 } 1363 } 1364 pages_boot(); 1365 if (base_boot()) 1366 return (true); 1367 if (chunk_boot()) 1368 return (true); 1369 if (ctl_boot()) 1370 return (true); 1371 if (config_prof) 1372 prof_boot1(); 1373 arena_boot(); 1374 if (config_tcache && tcache_boot(TSDN_NULL)) 1375 return (true); 1376 if (malloc_mutex_init(&arenas_lock, "arenas", WITNESS_RANK_ARENAS)) 1377 return (true); 1378 /* 1379 * Create enough scaffolding to allow recursive allocation in 1380 * malloc_ncpus(). 1381 */ 1382 narenas_auto = 1; 1383 narenas_total_set(narenas_auto); 1384 arenas = &a0; 1385 memset(arenas, 0, sizeof(arena_t *) * narenas_auto); 1386 /* 1387 * Initialize one arena here. The rest are lazily created in 1388 * arena_choose_hard(). 1389 */ 1390 if (arena_init(TSDN_NULL, 0) == NULL) 1391 return (true); 1392 1393 malloc_init_state = malloc_init_a0_initialized; 1394 1395 return (false); 1396} 1397 1398static bool 1399malloc_init_hard_a0(void) 1400{ 1401 bool ret; 1402 1403 malloc_mutex_lock(TSDN_NULL, &init_lock); 1404 ret = malloc_init_hard_a0_locked(); 1405 malloc_mutex_unlock(TSDN_NULL, &init_lock); 1406 return (ret); 1407} 1408 1409/* Initialize data structures which may trigger recursive allocation. */ 1410static bool 1411malloc_init_hard_recursible(void) 1412{ 1413 1414 malloc_init_state = malloc_init_recursible; 1415 1416 ncpus = malloc_ncpus(); 1417 1418#if (defined(JEMALLOC_HAVE_PTHREAD_ATFORK) && !defined(JEMALLOC_MUTEX_INIT_CB) \ 1419 && !defined(JEMALLOC_ZONE) && !defined(_WIN32) && \ 1420 !defined(__native_client__)) 1421 /* LinuxThreads' pthread_atfork() allocates. */ 1422 if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent, 1423 jemalloc_postfork_child) != 0) { 1424 malloc_write("<jemalloc>: Error in pthread_atfork()\n"); 1425 if (opt_abort) 1426 abort(); 1427 return (true); 1428 } 1429#endif 1430 1431 return (false); 1432} 1433 1434static bool 1435malloc_init_hard_finish(tsdn_t *tsdn) 1436{ 1437 1438 if (malloc_mutex_boot()) 1439 return (true); 1440 1441 if (opt_narenas == 0) { 1442 /* 1443 * For SMP systems, create more than one arena per CPU by 1444 * default. 1445 */ 1446 if (ncpus > 1) 1447 opt_narenas = ncpus << 2; 1448 else 1449 opt_narenas = 1; 1450 } 1451#if defined(ANDROID_MAX_ARENAS) 1452 /* Never create more than MAX_ARENAS arenas regardless of num_cpus. 1453 * Extra arenas use more PSS and are not very useful unless 1454 * lots of threads are allocing/freeing at the same time. 1455 */ 1456 if (opt_narenas > ANDROID_MAX_ARENAS) 1457 opt_narenas = ANDROID_MAX_ARENAS; 1458#endif 1459 narenas_auto = opt_narenas; 1460 /* 1461 * Limit the number of arenas to the indexing range of MALLOCX_ARENA(). 1462 */ 1463 if (narenas_auto > MALLOCX_ARENA_MAX) { 1464 narenas_auto = MALLOCX_ARENA_MAX; 1465 malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n", 1466 narenas_auto); 1467 } 1468 narenas_total_set(narenas_auto); 1469 1470 /* Allocate and initialize arenas. */ 1471 arenas = (arena_t **)base_alloc(tsdn, sizeof(arena_t *) * 1472 (MALLOCX_ARENA_MAX+1)); 1473 if (arenas == NULL) 1474 return (true); 1475 /* Copy the pointer to the one arena that was already initialized. */ 1476 arena_set(0, a0); 1477 1478 malloc_init_state = malloc_init_initialized; 1479 malloc_slow_flag_init(); 1480 1481 return (false); 1482} 1483 1484static bool 1485malloc_init_hard(void) 1486{ 1487 tsd_t *tsd; 1488 1489#if defined(_WIN32) && _WIN32_WINNT < 0x0600 1490 _init_init_lock(); 1491#endif 1492 malloc_mutex_lock(TSDN_NULL, &init_lock); 1493 if (!malloc_init_hard_needed()) { 1494 malloc_mutex_unlock(TSDN_NULL, &init_lock); 1495 return (false); 1496 } 1497 1498 if (malloc_init_state != malloc_init_a0_initialized && 1499 malloc_init_hard_a0_locked()) { 1500 malloc_mutex_unlock(TSDN_NULL, &init_lock); 1501 return (true); 1502 } 1503 1504 malloc_mutex_unlock(TSDN_NULL, &init_lock); 1505 /* Recursive allocation relies on functional tsd. */ 1506 tsd = malloc_tsd_boot0(); 1507 if (tsd == NULL) 1508 return (true); 1509 if (malloc_init_hard_recursible()) 1510 return (true); 1511 malloc_mutex_lock(tsd_tsdn(tsd), &init_lock); 1512 1513 if (config_prof && prof_boot2(tsd)) { 1514 malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock); 1515 return (true); 1516 } 1517 1518 if (malloc_init_hard_finish(tsd_tsdn(tsd))) { 1519 malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock); 1520 return (true); 1521 } 1522 1523 malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock); 1524 malloc_tsd_boot1(); 1525 return (false); 1526} 1527 1528/* 1529 * End initialization functions. 1530 */ 1531/******************************************************************************/ 1532/* 1533 * Begin malloc(3)-compatible functions. 1534 */ 1535 1536static void * 1537ialloc_prof_sample(tsd_t *tsd, size_t usize, szind_t ind, bool zero, 1538 prof_tctx_t *tctx, bool slow_path) 1539{ 1540 void *p; 1541 1542 if (tctx == NULL) 1543 return (NULL); 1544 if (usize <= SMALL_MAXCLASS) { 1545 szind_t ind_large = size2index(LARGE_MINCLASS); 1546 p = ialloc(tsd, LARGE_MINCLASS, ind_large, zero, slow_path); 1547 if (p == NULL) 1548 return (NULL); 1549 arena_prof_promoted(tsd_tsdn(tsd), p, usize); 1550 } else 1551 p = ialloc(tsd, usize, ind, zero, slow_path); 1552 1553 return (p); 1554} 1555 1556JEMALLOC_ALWAYS_INLINE_C void * 1557ialloc_prof(tsd_t *tsd, size_t usize, szind_t ind, bool zero, bool slow_path) 1558{ 1559 void *p; 1560 prof_tctx_t *tctx; 1561 1562 tctx = prof_alloc_prep(tsd, usize, prof_active_get_unlocked(), true); 1563 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) 1564 p = ialloc_prof_sample(tsd, usize, ind, zero, tctx, slow_path); 1565 else 1566 p = ialloc(tsd, usize, ind, zero, slow_path); 1567 if (unlikely(p == NULL)) { 1568 prof_alloc_rollback(tsd, tctx, true); 1569 return (NULL); 1570 } 1571 prof_malloc(tsd_tsdn(tsd), p, usize, tctx); 1572 1573 return (p); 1574} 1575 1576/* 1577 * ialloc_body() is inlined so that fast and slow paths are generated separately 1578 * with statically known slow_path. 1579 * 1580 * This function guarantees that *tsdn is non-NULL on success. 1581 */ 1582JEMALLOC_ALWAYS_INLINE_C void * 1583ialloc_body(size_t size, bool zero, tsdn_t **tsdn, size_t *usize, 1584 bool slow_path) 1585{ 1586 tsd_t *tsd; 1587 szind_t ind; 1588 1589 if (slow_path && unlikely(malloc_init())) { 1590 *tsdn = NULL; 1591 return (NULL); 1592 } 1593 1594 tsd = tsd_fetch(); 1595 *tsdn = tsd_tsdn(tsd); 1596 witness_assert_lockless(tsd_tsdn(tsd)); 1597 1598 ind = size2index(size); 1599 if (unlikely(ind >= NSIZES)) 1600 return (NULL); 1601 1602 if (config_stats || (config_prof && opt_prof) || (slow_path && 1603 config_valgrind && unlikely(in_valgrind))) { 1604 *usize = index2size(ind); 1605 assert(*usize > 0 && *usize <= HUGE_MAXCLASS); 1606 } 1607 1608 if (config_prof && opt_prof) 1609 return (ialloc_prof(tsd, *usize, ind, zero, slow_path)); 1610 1611 return (ialloc(tsd, size, ind, zero, slow_path)); 1612} 1613 1614JEMALLOC_ALWAYS_INLINE_C void 1615ialloc_post_check(void *ret, tsdn_t *tsdn, size_t usize, const char *func, 1616 bool update_errno, bool slow_path) 1617{ 1618 1619 assert(!tsdn_null(tsdn) || ret == NULL); 1620 1621 if (unlikely(ret == NULL)) { 1622 if (slow_path && config_xmalloc && unlikely(opt_xmalloc)) { 1623 malloc_printf("<jemalloc>: Error in %s(): out of " 1624 "memory\n", func); 1625 abort(); 1626 } 1627 if (update_errno) 1628 set_errno(ENOMEM); 1629 } 1630 if (config_stats && likely(ret != NULL)) { 1631 assert(usize == isalloc(tsdn, ret, config_prof)); 1632 *tsd_thread_allocatedp_get(tsdn_tsd(tsdn)) += usize; 1633 } 1634 witness_assert_lockless(tsdn); 1635} 1636 1637JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 1638void JEMALLOC_NOTHROW * 1639JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) 1640je_malloc(size_t size) 1641{ 1642 void *ret; 1643 tsdn_t *tsdn; 1644 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1645 1646 if (size == 0) 1647 size = 1; 1648 1649 if (likely(!malloc_slow)) { 1650 ret = ialloc_body(size, false, &tsdn, &usize, false); 1651 ialloc_post_check(ret, tsdn, usize, "malloc", true, false); 1652 } else { 1653 ret = ialloc_body(size, false, &tsdn, &usize, true); 1654 ialloc_post_check(ret, tsdn, usize, "malloc", true, true); 1655 UTRACE(0, size, ret); 1656 JEMALLOC_VALGRIND_MALLOC(ret != NULL, tsdn, ret, usize, false); 1657 } 1658 1659 return (ret); 1660} 1661 1662static void * 1663imemalign_prof_sample(tsd_t *tsd, size_t alignment, size_t usize, 1664 prof_tctx_t *tctx) 1665{ 1666 void *p; 1667 1668 if (tctx == NULL) 1669 return (NULL); 1670 if (usize <= SMALL_MAXCLASS) { 1671 assert(sa2u(LARGE_MINCLASS, alignment) == LARGE_MINCLASS); 1672 p = ipalloc(tsd, LARGE_MINCLASS, alignment, false); 1673 if (p == NULL) 1674 return (NULL); 1675 arena_prof_promoted(tsd_tsdn(tsd), p, usize); 1676 } else 1677 p = ipalloc(tsd, usize, alignment, false); 1678 1679 return (p); 1680} 1681 1682JEMALLOC_ALWAYS_INLINE_C void * 1683imemalign_prof(tsd_t *tsd, size_t alignment, size_t usize) 1684{ 1685 void *p; 1686 prof_tctx_t *tctx; 1687 1688 tctx = prof_alloc_prep(tsd, usize, prof_active_get_unlocked(), true); 1689 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) 1690 p = imemalign_prof_sample(tsd, alignment, usize, tctx); 1691 else 1692 p = ipalloc(tsd, usize, alignment, false); 1693 if (unlikely(p == NULL)) { 1694 prof_alloc_rollback(tsd, tctx, true); 1695 return (NULL); 1696 } 1697 prof_malloc(tsd_tsdn(tsd), p, usize, tctx); 1698 1699 return (p); 1700} 1701 1702JEMALLOC_ATTR(nonnull(1)) 1703static int 1704imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment) 1705{ 1706 int ret; 1707 tsd_t *tsd; 1708 size_t usize; 1709 void *result; 1710 1711 assert(min_alignment != 0); 1712 1713 if (unlikely(malloc_init())) { 1714 tsd = NULL; 1715 result = NULL; 1716 goto label_oom; 1717 } 1718 tsd = tsd_fetch(); 1719 witness_assert_lockless(tsd_tsdn(tsd)); 1720 if (size == 0) 1721 size = 1; 1722 1723 /* Make sure that alignment is a large enough power of 2. */ 1724 if (unlikely(((alignment - 1) & alignment) != 0 1725 || (alignment < min_alignment))) { 1726 if (config_xmalloc && unlikely(opt_xmalloc)) { 1727 malloc_write("<jemalloc>: Error allocating " 1728 "aligned memory: invalid alignment\n"); 1729 abort(); 1730 } 1731 result = NULL; 1732 ret = EINVAL; 1733 goto label_return; 1734 } 1735 1736 usize = sa2u(size, alignment); 1737 if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) { 1738 result = NULL; 1739 goto label_oom; 1740 } 1741 1742 if (config_prof && opt_prof) 1743 result = imemalign_prof(tsd, alignment, usize); 1744 else 1745 result = ipalloc(tsd, usize, alignment, false); 1746 if (unlikely(result == NULL)) 1747 goto label_oom; 1748 assert(((uintptr_t)result & (alignment - 1)) == ZU(0)); 1749 1750 *memptr = result; 1751 ret = 0; 1752label_return: 1753 if (config_stats && likely(result != NULL)) { 1754 assert(usize == isalloc(tsd_tsdn(tsd), result, config_prof)); 1755 *tsd_thread_allocatedp_get(tsd) += usize; 1756 } 1757 UTRACE(0, size, result); 1758 JEMALLOC_VALGRIND_MALLOC(result != NULL, tsd_tsdn(tsd), result, usize, 1759 false); 1760 witness_assert_lockless(tsd_tsdn(tsd)); 1761 return (ret); 1762label_oom: 1763 assert(result == NULL); 1764 if (config_xmalloc && unlikely(opt_xmalloc)) { 1765 malloc_write("<jemalloc>: Error allocating aligned memory: " 1766 "out of memory\n"); 1767 abort(); 1768 } 1769 ret = ENOMEM; 1770 witness_assert_lockless(tsd_tsdn(tsd)); 1771 goto label_return; 1772} 1773 1774JEMALLOC_EXPORT int JEMALLOC_NOTHROW 1775JEMALLOC_ATTR(nonnull(1)) 1776je_posix_memalign(void **memptr, size_t alignment, size_t size) 1777{ 1778 int ret; 1779 1780 ret = imemalign(memptr, alignment, size, sizeof(void *)); 1781 1782 return (ret); 1783} 1784 1785JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 1786void JEMALLOC_NOTHROW * 1787JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(2) 1788je_aligned_alloc(size_t alignment, size_t size) 1789{ 1790 void *ret; 1791 int err; 1792 1793 if (unlikely((err = imemalign(&ret, alignment, size, 1)) != 0)) { 1794 ret = NULL; 1795 set_errno(err); 1796 } 1797 1798 return (ret); 1799} 1800 1801JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 1802void JEMALLOC_NOTHROW * 1803JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2) 1804je_calloc(size_t num, size_t size) 1805{ 1806 void *ret; 1807 tsdn_t *tsdn; 1808 size_t num_size; 1809 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1810 1811 num_size = num * size; 1812 if (unlikely(num_size == 0)) { 1813 if (num == 0 || size == 0) 1814 num_size = 1; 1815 else 1816 num_size = HUGE_MAXCLASS + 1; /* Trigger OOM. */ 1817 /* 1818 * Try to avoid division here. We know that it isn't possible to 1819 * overflow during multiplication if neither operand uses any of the 1820 * most significant half of the bits in a size_t. 1821 */ 1822 } else if (unlikely(((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 1823 2))) && (num_size / size != num))) 1824 num_size = HUGE_MAXCLASS + 1; /* size_t overflow. */ 1825 1826 if (likely(!malloc_slow)) { 1827 ret = ialloc_body(num_size, true, &tsdn, &usize, false); 1828 ialloc_post_check(ret, tsdn, usize, "calloc", true, false); 1829 } else { 1830 ret = ialloc_body(num_size, true, &tsdn, &usize, true); 1831 ialloc_post_check(ret, tsdn, usize, "calloc", true, true); 1832 UTRACE(0, num_size, ret); 1833 JEMALLOC_VALGRIND_MALLOC(ret != NULL, tsdn, ret, usize, true); 1834 } 1835 1836 return (ret); 1837} 1838 1839static void * 1840irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, 1841 prof_tctx_t *tctx) 1842{ 1843 void *p; 1844 1845 if (tctx == NULL) 1846 return (NULL); 1847 if (usize <= SMALL_MAXCLASS) { 1848 p = iralloc(tsd, old_ptr, old_usize, LARGE_MINCLASS, 0, false); 1849 if (p == NULL) 1850 return (NULL); 1851 arena_prof_promoted(tsd_tsdn(tsd), p, usize); 1852 } else 1853 p = iralloc(tsd, old_ptr, old_usize, usize, 0, false); 1854 1855 return (p); 1856} 1857 1858JEMALLOC_ALWAYS_INLINE_C void * 1859irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize) 1860{ 1861 void *p; 1862 bool prof_active; 1863 prof_tctx_t *old_tctx, *tctx; 1864 1865 prof_active = prof_active_get_unlocked(); 1866 old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr); 1867 tctx = prof_alloc_prep(tsd, usize, prof_active, true); 1868 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) 1869 p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx); 1870 else 1871 p = iralloc(tsd, old_ptr, old_usize, usize, 0, false); 1872 if (unlikely(p == NULL)) { 1873 prof_alloc_rollback(tsd, tctx, true); 1874 return (NULL); 1875 } 1876 prof_realloc(tsd, p, usize, tctx, prof_active, true, old_ptr, old_usize, 1877 old_tctx); 1878 1879 return (p); 1880} 1881 1882JEMALLOC_INLINE_C void 1883ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) 1884{ 1885 size_t usize; 1886 UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); 1887 1888 witness_assert_lockless(tsd_tsdn(tsd)); 1889 1890 assert(ptr != NULL); 1891 assert(malloc_initialized() || IS_INITIALIZER); 1892 1893 if (config_prof && opt_prof) { 1894 usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); 1895 prof_free(tsd, ptr, usize); 1896 } else if (config_stats || config_valgrind) 1897 usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); 1898 if (config_stats) 1899 *tsd_thread_deallocatedp_get(tsd) += usize; 1900 1901 if (likely(!slow_path)) 1902 iqalloc(tsd, ptr, tcache, false); 1903 else { 1904 if (config_valgrind && unlikely(in_valgrind)) 1905 rzsize = p2rz(tsd_tsdn(tsd), ptr); 1906 iqalloc(tsd, ptr, tcache, true); 1907 JEMALLOC_VALGRIND_FREE(ptr, rzsize); 1908 } 1909} 1910 1911JEMALLOC_INLINE_C void 1912isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) 1913{ 1914 UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); 1915 1916 witness_assert_lockless(tsd_tsdn(tsd)); 1917 1918 assert(ptr != NULL); 1919 assert(malloc_initialized() || IS_INITIALIZER); 1920 1921 if (config_prof && opt_prof) 1922 prof_free(tsd, ptr, usize); 1923 if (config_stats) 1924 *tsd_thread_deallocatedp_get(tsd) += usize; 1925 if (config_valgrind && unlikely(in_valgrind)) 1926 rzsize = p2rz(tsd_tsdn(tsd), ptr); 1927 isqalloc(tsd, ptr, usize, tcache, slow_path); 1928 JEMALLOC_VALGRIND_FREE(ptr, rzsize); 1929} 1930 1931JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 1932void JEMALLOC_NOTHROW * 1933JEMALLOC_ALLOC_SIZE(2) 1934je_realloc(void *ptr, size_t size) 1935{ 1936 void *ret; 1937 tsdn_t *tsdn JEMALLOC_CC_SILENCE_INIT(NULL); 1938 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1939 size_t old_usize = 0; 1940 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 1941 1942 if (unlikely(size == 0)) { 1943 if (ptr != NULL) { 1944 tsd_t *tsd; 1945 1946 /* realloc(ptr, 0) is equivalent to free(ptr). */ 1947 UTRACE(ptr, 0, 0); 1948 tsd = tsd_fetch(); 1949 ifree(tsd, ptr, tcache_get(tsd, false), true); 1950 return (NULL); 1951 } 1952 size = 1; 1953 } 1954 1955 if (likely(ptr != NULL)) { 1956 tsd_t *tsd; 1957 1958 assert(malloc_initialized() || IS_INITIALIZER); 1959 malloc_thread_init(); 1960 tsd = tsd_fetch(); 1961 1962 witness_assert_lockless(tsd_tsdn(tsd)); 1963 1964 old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); 1965 if (config_valgrind && unlikely(in_valgrind)) { 1966 old_rzsize = config_prof ? p2rz(tsd_tsdn(tsd), ptr) : 1967 u2rz(old_usize); 1968 } 1969 1970 if (config_prof && opt_prof) { 1971 usize = s2u(size); 1972 ret = unlikely(usize == 0 || usize > HUGE_MAXCLASS) ? 1973 NULL : irealloc_prof(tsd, ptr, old_usize, usize); 1974 } else { 1975 if (config_stats || (config_valgrind && 1976 unlikely(in_valgrind))) 1977 usize = s2u(size); 1978 ret = iralloc(tsd, ptr, old_usize, size, 0, false); 1979 } 1980 tsdn = tsd_tsdn(tsd); 1981 } else { 1982 /* realloc(NULL, size) is equivalent to malloc(size). */ 1983 if (likely(!malloc_slow)) 1984 ret = ialloc_body(size, false, &tsdn, &usize, false); 1985 else 1986 ret = ialloc_body(size, false, &tsdn, &usize, true); 1987 assert(!tsdn_null(tsdn) || ret == NULL); 1988 } 1989 1990 if (unlikely(ret == NULL)) { 1991 if (config_xmalloc && unlikely(opt_xmalloc)) { 1992 malloc_write("<jemalloc>: Error in realloc(): " 1993 "out of memory\n"); 1994 abort(); 1995 } 1996 set_errno(ENOMEM); 1997 } 1998 if (config_stats && likely(ret != NULL)) { 1999 tsd_t *tsd; 2000 2001 assert(usize == isalloc(tsdn, ret, config_prof)); 2002 tsd = tsdn_tsd(tsdn); 2003 *tsd_thread_allocatedp_get(tsd) += usize; 2004 *tsd_thread_deallocatedp_get(tsd) += old_usize; 2005 } 2006 UTRACE(ptr, size, ret); 2007 JEMALLOC_VALGRIND_REALLOC(maybe, tsdn, ret, usize, maybe, ptr, 2008 old_usize, old_rzsize, maybe, false); 2009 witness_assert_lockless(tsdn); 2010 return (ret); 2011} 2012 2013JEMALLOC_EXPORT void JEMALLOC_NOTHROW 2014je_free(void *ptr) 2015{ 2016 2017 UTRACE(ptr, 0, 0); 2018 if (likely(ptr != NULL)) { 2019 tsd_t *tsd = tsd_fetch(); 2020 witness_assert_lockless(tsd_tsdn(tsd)); 2021 if (likely(!malloc_slow)) 2022 ifree(tsd, ptr, tcache_get(tsd, false), false); 2023 else 2024 ifree(tsd, ptr, tcache_get(tsd, false), true); 2025 witness_assert_lockless(tsd_tsdn(tsd)); 2026 } 2027} 2028 2029/* 2030 * End malloc(3)-compatible functions. 2031 */ 2032/******************************************************************************/ 2033/* 2034 * Begin non-standard override functions. 2035 */ 2036 2037#ifdef JEMALLOC_OVERRIDE_MEMALIGN 2038JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 2039void JEMALLOC_NOTHROW * 2040JEMALLOC_ATTR(malloc) 2041je_memalign(size_t alignment, size_t size) 2042{ 2043 void *ret JEMALLOC_CC_SILENCE_INIT(NULL); 2044 if (unlikely(imemalign(&ret, alignment, size, 1) != 0)) 2045 ret = NULL; 2046 return (ret); 2047} 2048#endif 2049 2050#ifdef JEMALLOC_OVERRIDE_VALLOC 2051JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 2052void JEMALLOC_NOTHROW * 2053JEMALLOC_ATTR(malloc) 2054je_valloc(size_t size) 2055{ 2056 void *ret JEMALLOC_CC_SILENCE_INIT(NULL); 2057 if (unlikely(imemalign(&ret, PAGE, size, 1) != 0)) 2058 ret = NULL; 2059 return (ret); 2060} 2061#endif 2062 2063/* 2064 * is_malloc(je_malloc) is some macro magic to detect if jemalloc_defs.h has 2065 * #define je_malloc malloc 2066 */ 2067#define malloc_is_malloc 1 2068#define is_malloc_(a) malloc_is_ ## a 2069#define is_malloc(a) is_malloc_(a) 2070 2071#if ((is_malloc(je_malloc) == 1) && defined(JEMALLOC_GLIBC_MALLOC_HOOK)) 2072/* 2073 * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible 2074 * to inconsistently reference libc's malloc(3)-compatible functions 2075 * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541). 2076 * 2077 * These definitions interpose hooks in glibc. The functions are actually 2078 * passed an extra argument for the caller return address, which will be 2079 * ignored. 2080 */ 2081JEMALLOC_EXPORT void (*__free_hook)(void *ptr) = je_free; 2082JEMALLOC_EXPORT void *(*__malloc_hook)(size_t size) = je_malloc; 2083JEMALLOC_EXPORT void *(*__realloc_hook)(void *ptr, size_t size) = je_realloc; 2084# ifdef JEMALLOC_GLIBC_MEMALIGN_HOOK 2085JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) = 2086 je_memalign; 2087# endif 2088 2089#ifdef CPU_COUNT 2090/* 2091 * To enable static linking with glibc, the libc specific malloc interface must 2092 * be implemented also, so none of glibc's malloc.o functions are added to the 2093 * link. 2094 */ 2095#define ALIAS(je_fn) __attribute__((alias (#je_fn), used)) 2096/* To force macro expansion of je_ prefix before stringification. */ 2097#define PREALIAS(je_fn) ALIAS(je_fn) 2098void *__libc_malloc(size_t size) PREALIAS(je_malloc); 2099void __libc_free(void* ptr) PREALIAS(je_free); 2100void *__libc_realloc(void* ptr, size_t size) PREALIAS(je_realloc); 2101void *__libc_calloc(size_t n, size_t size) PREALIAS(je_calloc); 2102void *__libc_memalign(size_t align, size_t s) PREALIAS(je_memalign); 2103void *__libc_valloc(size_t size) PREALIAS(je_valloc); 2104int __posix_memalign(void** r, size_t a, size_t s) 2105 PREALIAS(je_posix_memalign); 2106#undef PREALIAS 2107#undef ALIAS 2108 2109#endif 2110 2111#endif 2112 2113/* 2114 * End non-standard override functions. 2115 */ 2116/******************************************************************************/ 2117/* 2118 * Begin non-standard functions. 2119 */ 2120 2121JEMALLOC_ALWAYS_INLINE_C bool 2122imallocx_flags_decode(tsd_t *tsd, size_t size, int flags, size_t *usize, 2123 size_t *alignment, bool *zero, tcache_t **tcache, arena_t **arena) 2124{ 2125 2126 if ((flags & MALLOCX_LG_ALIGN_MASK) == 0) { 2127 *alignment = 0; 2128 *usize = s2u(size); 2129 } else { 2130 *alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags); 2131 *usize = sa2u(size, *alignment); 2132 } 2133 if (unlikely(*usize == 0 || *usize > HUGE_MAXCLASS)) 2134 return (true); 2135 *zero = MALLOCX_ZERO_GET(flags); 2136 if ((flags & MALLOCX_TCACHE_MASK) != 0) { 2137 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) 2138 *tcache = NULL; 2139 else 2140 *tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); 2141 } else 2142 *tcache = tcache_get(tsd, true); 2143 if ((flags & MALLOCX_ARENA_MASK) != 0) { 2144 unsigned arena_ind = MALLOCX_ARENA_GET(flags); 2145 *arena = arena_get(tsd_tsdn(tsd), arena_ind, true); 2146 if (unlikely(*arena == NULL)) 2147 return (true); 2148 } else 2149 *arena = NULL; 2150 return (false); 2151} 2152 2153JEMALLOC_ALWAYS_INLINE_C void * 2154imallocx_flags(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, 2155 tcache_t *tcache, arena_t *arena, bool slow_path) 2156{ 2157 szind_t ind; 2158 2159 if (unlikely(alignment != 0)) 2160 return (ipalloct(tsdn, usize, alignment, zero, tcache, arena)); 2161 ind = size2index(usize); 2162 assert(ind < NSIZES); 2163 return (iallocztm(tsdn, usize, ind, zero, tcache, false, arena, 2164 slow_path)); 2165} 2166 2167static void * 2168imallocx_prof_sample(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, 2169 tcache_t *tcache, arena_t *arena, bool slow_path) 2170{ 2171 void *p; 2172 2173 if (usize <= SMALL_MAXCLASS) { 2174 assert(((alignment == 0) ? s2u(LARGE_MINCLASS) : 2175 sa2u(LARGE_MINCLASS, alignment)) == LARGE_MINCLASS); 2176 p = imallocx_flags(tsdn, LARGE_MINCLASS, alignment, zero, 2177 tcache, arena, slow_path); 2178 if (p == NULL) 2179 return (NULL); 2180 arena_prof_promoted(tsdn, p, usize); 2181 } else { 2182 p = imallocx_flags(tsdn, usize, alignment, zero, tcache, arena, 2183 slow_path); 2184 } 2185 2186 return (p); 2187} 2188 2189JEMALLOC_ALWAYS_INLINE_C void * 2190imallocx_prof(tsd_t *tsd, size_t size, int flags, size_t *usize, bool slow_path) 2191{ 2192 void *p; 2193 size_t alignment; 2194 bool zero; 2195 tcache_t *tcache; 2196 arena_t *arena; 2197 prof_tctx_t *tctx; 2198 2199 if (unlikely(imallocx_flags_decode(tsd, size, flags, usize, &alignment, 2200 &zero, &tcache, &arena))) 2201 return (NULL); 2202 tctx = prof_alloc_prep(tsd, *usize, prof_active_get_unlocked(), true); 2203 if (likely((uintptr_t)tctx == (uintptr_t)1U)) { 2204 p = imallocx_flags(tsd_tsdn(tsd), *usize, alignment, zero, 2205 tcache, arena, slow_path); 2206 } else if ((uintptr_t)tctx > (uintptr_t)1U) { 2207 p = imallocx_prof_sample(tsd_tsdn(tsd), *usize, alignment, zero, 2208 tcache, arena, slow_path); 2209 } else 2210 p = NULL; 2211 if (unlikely(p == NULL)) { 2212 prof_alloc_rollback(tsd, tctx, true); 2213 return (NULL); 2214 } 2215 prof_malloc(tsd_tsdn(tsd), p, *usize, tctx); 2216 2217 assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0)); 2218 return (p); 2219} 2220 2221JEMALLOC_ALWAYS_INLINE_C void * 2222imallocx_no_prof(tsd_t *tsd, size_t size, int flags, size_t *usize, 2223 bool slow_path) 2224{ 2225 void *p; 2226 size_t alignment; 2227 bool zero; 2228 tcache_t *tcache; 2229 arena_t *arena; 2230 2231 if (unlikely(imallocx_flags_decode(tsd, size, flags, usize, &alignment, 2232 &zero, &tcache, &arena))) 2233 return (NULL); 2234 p = imallocx_flags(tsd_tsdn(tsd), *usize, alignment, zero, tcache, 2235 arena, slow_path); 2236 assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0)); 2237 return (p); 2238} 2239 2240/* This function guarantees that *tsdn is non-NULL on success. */ 2241JEMALLOC_ALWAYS_INLINE_C void * 2242imallocx_body(size_t size, int flags, tsdn_t **tsdn, size_t *usize, 2243 bool slow_path) 2244{ 2245 tsd_t *tsd; 2246 2247 if (slow_path && unlikely(malloc_init())) { 2248 *tsdn = NULL; 2249 return (NULL); 2250 } 2251 2252 tsd = tsd_fetch(); 2253 *tsdn = tsd_tsdn(tsd); 2254 witness_assert_lockless(tsd_tsdn(tsd)); 2255 2256 if (likely(flags == 0)) { 2257 szind_t ind = size2index(size); 2258 if (unlikely(ind >= NSIZES)) 2259 return (NULL); 2260 if (config_stats || (config_prof && opt_prof) || (slow_path && 2261 config_valgrind && unlikely(in_valgrind))) { 2262 *usize = index2size(ind); 2263 assert(*usize > 0 && *usize <= HUGE_MAXCLASS); 2264 } 2265 2266 if (config_prof && opt_prof) { 2267 return (ialloc_prof(tsd, *usize, ind, false, 2268 slow_path)); 2269 } 2270 2271 return (ialloc(tsd, size, ind, false, slow_path)); 2272 } 2273 2274 if (config_prof && opt_prof) 2275 return (imallocx_prof(tsd, size, flags, usize, slow_path)); 2276 2277 return (imallocx_no_prof(tsd, size, flags, usize, slow_path)); 2278} 2279 2280JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 2281void JEMALLOC_NOTHROW * 2282JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) 2283je_mallocx(size_t size, int flags) 2284{ 2285 tsdn_t *tsdn; 2286 void *p; 2287 size_t usize; 2288 2289 assert(size != 0); 2290 2291 if (likely(!malloc_slow)) { 2292 p = imallocx_body(size, flags, &tsdn, &usize, false); 2293 ialloc_post_check(p, tsdn, usize, "mallocx", false, false); 2294 } else { 2295 p = imallocx_body(size, flags, &tsdn, &usize, true); 2296 ialloc_post_check(p, tsdn, usize, "mallocx", false, true); 2297 UTRACE(0, size, p); 2298 JEMALLOC_VALGRIND_MALLOC(p != NULL, tsdn, p, usize, 2299 MALLOCX_ZERO_GET(flags)); 2300 } 2301 2302 return (p); 2303} 2304 2305static void * 2306irallocx_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, 2307 size_t usize, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena, 2308 prof_tctx_t *tctx) 2309{ 2310 void *p; 2311 2312 if (tctx == NULL) 2313 return (NULL); 2314 if (usize <= SMALL_MAXCLASS) { 2315 p = iralloct(tsd, old_ptr, old_usize, LARGE_MINCLASS, alignment, 2316 zero, tcache, arena); 2317 if (p == NULL) 2318 return (NULL); 2319 arena_prof_promoted(tsd_tsdn(tsd), p, usize); 2320 } else { 2321 p = iralloct(tsd, old_ptr, old_usize, usize, alignment, zero, 2322 tcache, arena); 2323 } 2324 2325 return (p); 2326} 2327 2328JEMALLOC_ALWAYS_INLINE_C void * 2329irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size, 2330 size_t alignment, size_t *usize, bool zero, tcache_t *tcache, 2331 arena_t *arena) 2332{ 2333 void *p; 2334 bool prof_active; 2335 prof_tctx_t *old_tctx, *tctx; 2336 2337 prof_active = prof_active_get_unlocked(); 2338 old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr); 2339 tctx = prof_alloc_prep(tsd, *usize, prof_active, false); 2340 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { 2341 p = irallocx_prof_sample(tsd, old_ptr, old_usize, *usize, 2342 alignment, zero, tcache, arena, tctx); 2343 } else { 2344 p = iralloct(tsd, old_ptr, old_usize, size, alignment, zero, 2345 tcache, arena); 2346 } 2347 if (unlikely(p == NULL)) { 2348 prof_alloc_rollback(tsd, tctx, false); 2349 return (NULL); 2350 } 2351 2352 if (p == old_ptr && alignment != 0) { 2353 /* 2354 * The allocation did not move, so it is possible that the size 2355 * class is smaller than would guarantee the requested 2356 * alignment, and that the alignment constraint was 2357 * serendipitously satisfied. Additionally, old_usize may not 2358 * be the same as the current usize because of in-place large 2359 * reallocation. Therefore, query the actual value of usize. 2360 */ 2361 *usize = isalloc(tsd_tsdn(tsd), p, config_prof); 2362 } 2363 prof_realloc(tsd, p, *usize, tctx, prof_active, false, old_ptr, 2364 old_usize, old_tctx); 2365 2366 return (p); 2367} 2368 2369JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 2370void JEMALLOC_NOTHROW * 2371JEMALLOC_ALLOC_SIZE(2) 2372je_rallocx(void *ptr, size_t size, int flags) 2373{ 2374 void *p; 2375 tsd_t *tsd; 2376 size_t usize; 2377 size_t old_usize; 2378 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 2379 size_t alignment = MALLOCX_ALIGN_GET(flags); 2380 bool zero = flags & MALLOCX_ZERO; 2381 arena_t *arena; 2382 tcache_t *tcache; 2383 2384 assert(ptr != NULL); 2385 assert(size != 0); 2386 assert(malloc_initialized() || IS_INITIALIZER); 2387 malloc_thread_init(); 2388 tsd = tsd_fetch(); 2389 witness_assert_lockless(tsd_tsdn(tsd)); 2390 2391 if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { 2392 unsigned arena_ind = MALLOCX_ARENA_GET(flags); 2393 arena = arena_get(tsd_tsdn(tsd), arena_ind, true); 2394 if (unlikely(arena == NULL)) 2395 goto label_oom; 2396 } else 2397 arena = NULL; 2398 2399 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { 2400 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) 2401 tcache = NULL; 2402 else 2403 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); 2404 } else 2405 tcache = tcache_get(tsd, true); 2406 2407 old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); 2408 if (config_valgrind && unlikely(in_valgrind)) 2409 old_rzsize = u2rz(old_usize); 2410 2411 if (config_prof && opt_prof) { 2412 usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); 2413 if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) 2414 goto label_oom; 2415 p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize, 2416 zero, tcache, arena); 2417 if (unlikely(p == NULL)) 2418 goto label_oom; 2419 } else { 2420 p = iralloct(tsd, ptr, old_usize, size, alignment, zero, 2421 tcache, arena); 2422 if (unlikely(p == NULL)) 2423 goto label_oom; 2424 if (config_stats || (config_valgrind && unlikely(in_valgrind))) 2425 usize = isalloc(tsd_tsdn(tsd), p, config_prof); 2426 } 2427 assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0)); 2428 2429 if (config_stats) { 2430 *tsd_thread_allocatedp_get(tsd) += usize; 2431 *tsd_thread_deallocatedp_get(tsd) += old_usize; 2432 } 2433 UTRACE(ptr, size, p); 2434 JEMALLOC_VALGRIND_REALLOC(maybe, tsd_tsdn(tsd), p, usize, no, ptr, 2435 old_usize, old_rzsize, no, zero); 2436 witness_assert_lockless(tsd_tsdn(tsd)); 2437 return (p); 2438label_oom: 2439 if (config_xmalloc && unlikely(opt_xmalloc)) { 2440 malloc_write("<jemalloc>: Error in rallocx(): out of memory\n"); 2441 abort(); 2442 } 2443 UTRACE(ptr, size, 0); 2444 witness_assert_lockless(tsd_tsdn(tsd)); 2445 return (NULL); 2446} 2447 2448JEMALLOC_ALWAYS_INLINE_C size_t 2449ixallocx_helper(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size, 2450 size_t extra, size_t alignment, bool zero) 2451{ 2452 size_t usize; 2453 2454 if (ixalloc(tsdn, ptr, old_usize, size, extra, alignment, zero)) 2455 return (old_usize); 2456 usize = isalloc(tsdn, ptr, config_prof); 2457 2458 return (usize); 2459} 2460 2461static size_t 2462ixallocx_prof_sample(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size, 2463 size_t extra, size_t alignment, bool zero, prof_tctx_t *tctx) 2464{ 2465 size_t usize; 2466 2467 if (tctx == NULL) 2468 return (old_usize); 2469 usize = ixallocx_helper(tsdn, ptr, old_usize, size, extra, alignment, 2470 zero); 2471 2472 return (usize); 2473} 2474 2475JEMALLOC_ALWAYS_INLINE_C size_t 2476ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, 2477 size_t extra, size_t alignment, bool zero) 2478{ 2479 size_t usize_max, usize; 2480 bool prof_active; 2481 prof_tctx_t *old_tctx, *tctx; 2482 2483 prof_active = prof_active_get_unlocked(); 2484 old_tctx = prof_tctx_get(tsd_tsdn(tsd), ptr); 2485 /* 2486 * usize isn't knowable before ixalloc() returns when extra is non-zero. 2487 * Therefore, compute its maximum possible value and use that in 2488 * prof_alloc_prep() to decide whether to capture a backtrace. 2489 * prof_realloc() will use the actual usize to decide whether to sample. 2490 */ 2491 if (alignment == 0) { 2492 usize_max = s2u(size+extra); 2493 assert(usize_max > 0 && usize_max <= HUGE_MAXCLASS); 2494 } else { 2495 usize_max = sa2u(size+extra, alignment); 2496 if (unlikely(usize_max == 0 || usize_max > HUGE_MAXCLASS)) { 2497 /* 2498 * usize_max is out of range, and chances are that 2499 * allocation will fail, but use the maximum possible 2500 * value and carry on with prof_alloc_prep(), just in 2501 * case allocation succeeds. 2502 */ 2503 usize_max = HUGE_MAXCLASS; 2504 } 2505 } 2506 tctx = prof_alloc_prep(tsd, usize_max, prof_active, false); 2507 2508 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { 2509 usize = ixallocx_prof_sample(tsd_tsdn(tsd), ptr, old_usize, 2510 size, extra, alignment, zero, tctx); 2511 } else { 2512 usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size, 2513 extra, alignment, zero); 2514 } 2515 if (usize == old_usize) { 2516 prof_alloc_rollback(tsd, tctx, false); 2517 return (usize); 2518 } 2519 prof_realloc(tsd, ptr, usize, tctx, prof_active, false, ptr, old_usize, 2520 old_tctx); 2521 2522 return (usize); 2523} 2524 2525JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW 2526je_xallocx(void *ptr, size_t size, size_t extra, int flags) 2527{ 2528 tsd_t *tsd; 2529 size_t usize, old_usize; 2530 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 2531 size_t alignment = MALLOCX_ALIGN_GET(flags); 2532 bool zero = flags & MALLOCX_ZERO; 2533 2534 assert(ptr != NULL); 2535 assert(size != 0); 2536 assert(SIZE_T_MAX - size >= extra); 2537 assert(malloc_initialized() || IS_INITIALIZER); 2538 malloc_thread_init(); 2539 tsd = tsd_fetch(); 2540 witness_assert_lockless(tsd_tsdn(tsd)); 2541 2542 old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); 2543 2544 /* 2545 * The API explicitly absolves itself of protecting against (size + 2546 * extra) numerical overflow, but we may need to clamp extra to avoid 2547 * exceeding HUGE_MAXCLASS. 2548 * 2549 * Ordinarily, size limit checking is handled deeper down, but here we 2550 * have to check as part of (size + extra) clamping, since we need the 2551 * clamped value in the above helper functions. 2552 */ 2553 if (unlikely(size > HUGE_MAXCLASS)) { 2554 usize = old_usize; 2555 goto label_not_resized; 2556 } 2557 if (unlikely(HUGE_MAXCLASS - size < extra)) 2558 extra = HUGE_MAXCLASS - size; 2559 2560 if (config_valgrind && unlikely(in_valgrind)) 2561 old_rzsize = u2rz(old_usize); 2562 2563 if (config_prof && opt_prof) { 2564 usize = ixallocx_prof(tsd, ptr, old_usize, size, extra, 2565 alignment, zero); 2566 } else { 2567 usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size, 2568 extra, alignment, zero); 2569 } 2570 if (unlikely(usize == old_usize)) 2571 goto label_not_resized; 2572 2573 if (config_stats) { 2574 *tsd_thread_allocatedp_get(tsd) += usize; 2575 *tsd_thread_deallocatedp_get(tsd) += old_usize; 2576 } 2577 JEMALLOC_VALGRIND_REALLOC(no, tsd_tsdn(tsd), ptr, usize, no, ptr, 2578 old_usize, old_rzsize, no, zero); 2579label_not_resized: 2580 UTRACE(ptr, size, ptr); 2581 witness_assert_lockless(tsd_tsdn(tsd)); 2582 return (usize); 2583} 2584 2585JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW 2586JEMALLOC_ATTR(pure) 2587je_sallocx(const void *ptr, int flags) 2588{ 2589 size_t usize; 2590 tsdn_t *tsdn; 2591 2592 assert(malloc_initialized() || IS_INITIALIZER); 2593 malloc_thread_init(); 2594 2595 tsdn = tsdn_fetch(); 2596 witness_assert_lockless(tsdn); 2597 2598 if (config_ivsalloc) 2599 usize = ivsalloc(tsdn, ptr, config_prof); 2600 else 2601 usize = isalloc(tsdn, ptr, config_prof); 2602 2603 witness_assert_lockless(tsdn); 2604 return (usize); 2605} 2606 2607JEMALLOC_EXPORT void JEMALLOC_NOTHROW 2608je_dallocx(void *ptr, int flags) 2609{ 2610 tsd_t *tsd; 2611 tcache_t *tcache; 2612 2613 assert(ptr != NULL); 2614 assert(malloc_initialized() || IS_INITIALIZER); 2615 2616 tsd = tsd_fetch(); 2617 witness_assert_lockless(tsd_tsdn(tsd)); 2618 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { 2619 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) 2620 tcache = NULL; 2621 else 2622 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); 2623 } else 2624 tcache = tcache_get(tsd, false); 2625 2626 UTRACE(ptr, 0, 0); 2627 if (likely(!malloc_slow)) 2628 ifree(tsd, ptr, tcache, false); 2629 else 2630 ifree(tsd, ptr, tcache, true); 2631 witness_assert_lockless(tsd_tsdn(tsd)); 2632} 2633 2634JEMALLOC_ALWAYS_INLINE_C size_t 2635inallocx(tsdn_t *tsdn, size_t size, int flags) 2636{ 2637 size_t usize; 2638 2639 witness_assert_lockless(tsdn); 2640 2641 if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0)) 2642 usize = s2u(size); 2643 else 2644 usize = sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags)); 2645 witness_assert_lockless(tsdn); 2646 return (usize); 2647} 2648 2649JEMALLOC_EXPORT void JEMALLOC_NOTHROW 2650je_sdallocx(void *ptr, size_t size, int flags) 2651{ 2652 tsd_t *tsd; 2653 tcache_t *tcache; 2654 size_t usize; 2655 2656 assert(ptr != NULL); 2657 assert(malloc_initialized() || IS_INITIALIZER); 2658 tsd = tsd_fetch(); 2659 usize = inallocx(tsd_tsdn(tsd), size, flags); 2660 assert(usize == isalloc(tsd_tsdn(tsd), ptr, config_prof)); 2661 2662 witness_assert_lockless(tsd_tsdn(tsd)); 2663 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { 2664 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) 2665 tcache = NULL; 2666 else 2667 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); 2668 } else 2669 tcache = tcache_get(tsd, false); 2670 2671 UTRACE(ptr, 0, 0); 2672 if (likely(!malloc_slow)) 2673 isfree(tsd, ptr, usize, tcache, false); 2674 else 2675 isfree(tsd, ptr, usize, tcache, true); 2676 witness_assert_lockless(tsd_tsdn(tsd)); 2677} 2678 2679JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW 2680JEMALLOC_ATTR(pure) 2681je_nallocx(size_t size, int flags) 2682{ 2683 size_t usize; 2684 tsdn_t *tsdn; 2685 2686 assert(size != 0); 2687 2688 if (unlikely(malloc_init())) 2689 return (0); 2690 2691 tsdn = tsdn_fetch(); 2692 witness_assert_lockless(tsdn); 2693 2694 usize = inallocx(tsdn, size, flags); 2695 if (unlikely(usize > HUGE_MAXCLASS)) 2696 return (0); 2697 2698 witness_assert_lockless(tsdn); 2699 return (usize); 2700} 2701 2702JEMALLOC_EXPORT int JEMALLOC_NOTHROW 2703je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, 2704 size_t newlen) 2705{ 2706 int ret; 2707 tsd_t *tsd; 2708 2709 if (unlikely(malloc_init())) 2710 return (EAGAIN); 2711 2712 tsd = tsd_fetch(); 2713 witness_assert_lockless(tsd_tsdn(tsd)); 2714 ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen); 2715 witness_assert_lockless(tsd_tsdn(tsd)); 2716 return (ret); 2717} 2718 2719JEMALLOC_EXPORT int JEMALLOC_NOTHROW 2720je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) 2721{ 2722 int ret; 2723 tsdn_t *tsdn; 2724 2725 if (unlikely(malloc_init())) 2726 return (EAGAIN); 2727 2728 tsdn = tsdn_fetch(); 2729 witness_assert_lockless(tsdn); 2730 ret = ctl_nametomib(tsdn, name, mibp, miblenp); 2731 witness_assert_lockless(tsdn); 2732 return (ret); 2733} 2734 2735JEMALLOC_EXPORT int JEMALLOC_NOTHROW 2736je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, 2737 void *newp, size_t newlen) 2738{ 2739 int ret; 2740 tsd_t *tsd; 2741 2742 if (unlikely(malloc_init())) 2743 return (EAGAIN); 2744 2745 tsd = tsd_fetch(); 2746 witness_assert_lockless(tsd_tsdn(tsd)); 2747 ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen); 2748 witness_assert_lockless(tsd_tsdn(tsd)); 2749 return (ret); 2750} 2751 2752JEMALLOC_EXPORT void JEMALLOC_NOTHROW 2753je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, 2754 const char *opts) 2755{ 2756 tsdn_t *tsdn; 2757 2758 tsdn = tsdn_fetch(); 2759 witness_assert_lockless(tsdn); 2760 stats_print(write_cb, cbopaque, opts); 2761 witness_assert_lockless(tsdn); 2762} 2763 2764JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW 2765je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) 2766{ 2767 size_t ret; 2768 tsdn_t *tsdn; 2769 2770 assert(malloc_initialized() || IS_INITIALIZER); 2771 malloc_thread_init(); 2772 2773 tsdn = tsdn_fetch(); 2774 witness_assert_lockless(tsdn); 2775 2776 if (config_ivsalloc) 2777 ret = ivsalloc(tsdn, ptr, config_prof); 2778 else 2779 ret = (ptr == NULL) ? 0 : isalloc(tsdn, ptr, config_prof); 2780 2781 witness_assert_lockless(tsdn); 2782 return (ret); 2783} 2784 2785/* 2786 * End non-standard functions. 2787 */ 2788/******************************************************************************/ 2789/* 2790 * The following functions are used by threading libraries for protection of 2791 * malloc during fork(). 2792 */ 2793 2794/* 2795 * If an application creates a thread before doing any allocation in the main 2796 * thread, then calls fork(2) in the main thread followed by memory allocation 2797 * in the child process, a race can occur that results in deadlock within the 2798 * child: the main thread may have forked while the created thread had 2799 * partially initialized the allocator. Ordinarily jemalloc prevents 2800 * fork/malloc races via the following functions it registers during 2801 * initialization using pthread_atfork(), but of course that does no good if 2802 * the allocator isn't fully initialized at fork time. The following library 2803 * constructor is a partial solution to this problem. It may still be possible 2804 * to trigger the deadlock described above, but doing so would involve forking 2805 * via a library constructor that runs before jemalloc's runs. 2806 */ 2807#ifndef JEMALLOC_JET 2808JEMALLOC_ATTR(constructor) 2809static void 2810jemalloc_constructor(void) 2811{ 2812 2813 malloc_init(); 2814} 2815#endif 2816 2817#ifndef JEMALLOC_MUTEX_INIT_CB 2818void 2819jemalloc_prefork(void) 2820#else 2821JEMALLOC_EXPORT void 2822_malloc_prefork(void) 2823#endif 2824{ 2825 tsd_t *tsd; 2826 unsigned i, j, narenas; 2827 arena_t *arena; 2828 2829#ifdef JEMALLOC_MUTEX_INIT_CB 2830 if (!malloc_initialized()) 2831 return; 2832#endif 2833 assert(malloc_initialized()); 2834 2835 tsd = tsd_fetch(); 2836 2837 narenas = narenas_total_get(); 2838 2839 witness_prefork(tsd); 2840 /* Acquire all mutexes in a safe order. */ 2841 ctl_prefork(tsd_tsdn(tsd)); 2842 tcache_prefork(tsd_tsdn(tsd)); 2843 malloc_mutex_prefork(tsd_tsdn(tsd), &arenas_lock); 2844 prof_prefork0(tsd_tsdn(tsd)); 2845 for (i = 0; i < 3; i++) { 2846 for (j = 0; j < narenas; j++) { 2847 if ((arena = arena_get(tsd_tsdn(tsd), j, false)) != 2848 NULL) { 2849 switch (i) { 2850 case 0: 2851 arena_prefork0(tsd_tsdn(tsd), arena); 2852 break; 2853 case 1: 2854 arena_prefork1(tsd_tsdn(tsd), arena); 2855 break; 2856 case 2: 2857 arena_prefork2(tsd_tsdn(tsd), arena); 2858 break; 2859 default: not_reached(); 2860 } 2861 } 2862 } 2863 } 2864 base_prefork(tsd_tsdn(tsd)); 2865 for (i = 0; i < narenas; i++) { 2866 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) 2867 arena_prefork3(tsd_tsdn(tsd), arena); 2868 } 2869 prof_prefork1(tsd_tsdn(tsd)); 2870} 2871 2872#ifndef JEMALLOC_MUTEX_INIT_CB 2873void 2874jemalloc_postfork_parent(void) 2875#else 2876JEMALLOC_EXPORT void 2877_malloc_postfork(void) 2878#endif 2879{ 2880 tsd_t *tsd; 2881 unsigned i, narenas; 2882 2883#ifdef JEMALLOC_MUTEX_INIT_CB 2884 if (!malloc_initialized()) 2885 return; 2886#endif 2887 assert(malloc_initialized()); 2888 2889 tsd = tsd_fetch(); 2890 2891 witness_postfork_parent(tsd); 2892 /* Release all mutexes, now that fork() has completed. */ 2893 base_postfork_parent(tsd_tsdn(tsd)); 2894 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { 2895 arena_t *arena; 2896 2897 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) 2898 arena_postfork_parent(tsd_tsdn(tsd), arena); 2899 } 2900 prof_postfork_parent(tsd_tsdn(tsd)); 2901 malloc_mutex_postfork_parent(tsd_tsdn(tsd), &arenas_lock); 2902 tcache_postfork_parent(tsd_tsdn(tsd)); 2903 ctl_postfork_parent(tsd_tsdn(tsd)); 2904} 2905 2906void 2907jemalloc_postfork_child(void) 2908{ 2909 tsd_t *tsd; 2910 unsigned i, narenas; 2911 2912 assert(malloc_initialized()); 2913 2914 tsd = tsd_fetch(); 2915 2916 witness_postfork_child(tsd); 2917 /* Release all mutexes, now that fork() has completed. */ 2918 base_postfork_child(tsd_tsdn(tsd)); 2919 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { 2920 arena_t *arena; 2921 2922 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) 2923 arena_postfork_child(tsd_tsdn(tsd), arena); 2924 } 2925 prof_postfork_child(tsd_tsdn(tsd)); 2926 malloc_mutex_postfork_child(tsd_tsdn(tsd), &arenas_lock); 2927 tcache_postfork_child(tsd_tsdn(tsd)); 2928 ctl_postfork_child(tsd_tsdn(tsd)); 2929} 2930 2931/******************************************************************************/ 2932 2933/* ANDROID extension */ 2934#include "android_je_iterate.c" 2935#include "android_je_mallinfo.c" 2936/* End ANDROID extension */ 2937