jemalloc.c revision 9b75677e538836b284a0d26a593963187c24a153
1#define JEMALLOC_C_ 2#include "jemalloc/internal/jemalloc_internal.h" 3 4/******************************************************************************/ 5/* Data. */ 6 7/* Runtime configuration options. */ 8const char *je_malloc_conf JEMALLOC_ATTR(weak); 9bool opt_abort = 10#ifdef JEMALLOC_DEBUG 11 true 12#else 13 false 14#endif 15 ; 16bool opt_junk = 17#if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) 18 true 19#else 20 false 21#endif 22 ; 23size_t opt_quarantine = ZU(0); 24bool opt_redzone = false; 25bool opt_utrace = false; 26bool opt_xmalloc = false; 27bool opt_zero = false; 28size_t opt_narenas = 0; 29 30/* Initialized to true if the process is running inside Valgrind. */ 31bool in_valgrind; 32 33unsigned ncpus; 34 35/* Protects arenas initialization (arenas, narenas_total). */ 36static malloc_mutex_t arenas_lock; 37/* 38 * Arenas that are used to service external requests. Not all elements of the 39 * arenas array are necessarily used; arenas are created lazily as needed. 40 * 41 * arenas[0..narenas_auto) are used for automatic multiplexing of threads and 42 * arenas. arenas[narenas_auto..narenas_total) are only used if the application 43 * takes some action to create them and allocate from them. 44 */ 45static arena_t **arenas; 46static unsigned narenas_total; 47static arena_t *a0; /* arenas[0]; read-only after initialization. */ 48static unsigned narenas_auto; /* Read-only after initialization. */ 49 50/* Set to true once the allocator has been initialized. */ 51static bool malloc_initialized = false; 52 53JEMALLOC_ALIGNED(CACHELINE) 54const size_t index2size_tab[NSIZES] = { 55#define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \ 56 ((ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta)), 57 SIZE_CLASSES 58#undef SC 59}; 60 61JEMALLOC_ALIGNED(CACHELINE) 62const uint8_t size2index_tab[] = { 63#define S2B_3(i) i, 64#define S2B_4(i) S2B_3(i) S2B_3(i) 65#define S2B_5(i) S2B_4(i) S2B_4(i) 66#define S2B_6(i) S2B_5(i) S2B_5(i) 67#define S2B_7(i) S2B_6(i) S2B_6(i) 68#define S2B_8(i) S2B_7(i) S2B_7(i) 69#define S2B_9(i) S2B_8(i) S2B_8(i) 70#define S2B_10(i) S2B_9(i) S2B_9(i) 71#define S2B_11(i) S2B_10(i) S2B_10(i) 72#define S2B_no(i) 73#define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \ 74 S2B_##lg_delta_lookup(index) 75 SIZE_CLASSES 76#undef S2B_3 77#undef S2B_4 78#undef S2B_5 79#undef S2B_6 80#undef S2B_7 81#undef S2B_8 82#undef S2B_9 83#undef S2B_10 84#undef S2B_11 85#undef S2B_no 86#undef SC 87}; 88 89#ifdef JEMALLOC_THREADED_INIT 90/* Used to let the initializing thread recursively allocate. */ 91# define NO_INITIALIZER ((unsigned long)0) 92# define INITIALIZER pthread_self() 93# define IS_INITIALIZER (malloc_initializer == pthread_self()) 94static pthread_t malloc_initializer = NO_INITIALIZER; 95#else 96# define NO_INITIALIZER false 97# define INITIALIZER true 98# define IS_INITIALIZER malloc_initializer 99static bool malloc_initializer = NO_INITIALIZER; 100#endif 101 102/* Used to avoid initialization races. */ 103#ifdef _WIN32 104static malloc_mutex_t init_lock; 105 106JEMALLOC_ATTR(constructor) 107static void WINAPI 108_init_init_lock(void) 109{ 110 111 malloc_mutex_init(&init_lock); 112} 113 114#ifdef _MSC_VER 115# pragma section(".CRT$XCU", read) 116JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used) 117static const void (WINAPI *init_init_lock)(void) = _init_init_lock; 118#endif 119 120#else 121static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER; 122#endif 123 124typedef struct { 125 void *p; /* Input pointer (as in realloc(p, s)). */ 126 size_t s; /* Request size. */ 127 void *r; /* Result pointer. */ 128} malloc_utrace_t; 129 130#ifdef JEMALLOC_UTRACE 131# define UTRACE(a, b, c) do { \ 132 if (unlikely(opt_utrace)) { \ 133 int utrace_serrno = errno; \ 134 malloc_utrace_t ut; \ 135 ut.p = (a); \ 136 ut.s = (b); \ 137 ut.r = (c); \ 138 utrace(&ut, sizeof(ut)); \ 139 errno = utrace_serrno; \ 140 } \ 141} while (0) 142#else 143# define UTRACE(a, b, c) 144#endif 145 146/******************************************************************************/ 147/* 148 * Function prototypes for static functions that are referenced prior to 149 * definition. 150 */ 151 152static bool malloc_init_hard(void); 153 154/******************************************************************************/ 155/* 156 * Begin miscellaneous support functions. 157 */ 158 159JEMALLOC_ALWAYS_INLINE_C void 160malloc_thread_init(void) 161{ 162 163 /* 164 * TSD initialization can't be safely done as a side effect of 165 * deallocation, because it is possible for a thread to do nothing but 166 * deallocate its TLS data via free(), in which case writing to TLS 167 * would cause write-after-free memory corruption. The quarantine 168 * facility *only* gets used as a side effect of deallocation, so make 169 * a best effort attempt at initializing its TSD by hooking all 170 * allocation events. 171 */ 172 if (config_fill && unlikely(opt_quarantine)) 173 quarantine_alloc_hook(); 174} 175 176JEMALLOC_ALWAYS_INLINE_C bool 177malloc_init(void) 178{ 179 180 if (unlikely(!malloc_initialized) && malloc_init_hard()) 181 return (true); 182 malloc_thread_init(); 183 184 return (false); 185} 186 187/* 188 * The a0*() functions are used instead of i[mcd]alloc() in bootstrap-sensitive 189 * situations that cannot tolerate TLS variable access. These functions are 190 * also exposed for use in static binaries on FreeBSD, hence the old-style 191 * malloc() API. 192 */ 193 194arena_t * 195a0get(void) 196{ 197 198 assert(a0 != NULL); 199 return (a0); 200} 201 202static void * 203a0alloc(size_t size, bool zero) 204{ 205 void *ret; 206 207 if (unlikely(malloc_init())) 208 return (NULL); 209 210 if (size == 0) 211 size = 1; 212 213 if (size <= arena_maxclass) 214 ret = arena_malloc(NULL, a0get(), size, zero, false); 215 else 216 ret = huge_malloc(NULL, a0get(), size, zero, false); 217 218 return (ret); 219} 220 221void * 222a0malloc(size_t size) 223{ 224 225 return (a0alloc(size, false)); 226} 227 228void * 229a0calloc(size_t num, size_t size) 230{ 231 232 return (a0alloc(num * size, true)); 233} 234 235void 236a0free(void *ptr) 237{ 238 arena_chunk_t *chunk; 239 240 if (ptr == NULL) 241 return; 242 243 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 244 if (chunk != ptr) 245 arena_dalloc(NULL, chunk, ptr, false); 246 else 247 huge_dalloc(NULL, ptr, false); 248} 249 250/* Create a new arena and insert it into the arenas array at index ind. */ 251static arena_t * 252arena_init_locked(unsigned ind) 253{ 254 arena_t *arena; 255 256 /* Expand arenas if necessary. */ 257 assert(ind <= narenas_total); 258 if (ind == narenas_total) { 259 unsigned narenas_new = narenas_total + 1; 260 arena_t **arenas_new = 261 (arena_t **)a0malloc(CACHELINE_CEILING(narenas_new * 262 sizeof(arena_t *))); 263 if (arenas_new == NULL) 264 return (NULL); 265 memcpy(arenas_new, arenas, narenas_total * sizeof(arena_t *)); 266 arenas_new[ind] = NULL; 267 /* 268 * Deallocate only if arenas came from a0malloc() (not 269 * base_alloc()). 270 */ 271 if (narenas_total != narenas_auto) 272 a0free(arenas); 273 arenas = arenas_new; 274 narenas_total = narenas_new; 275 } 276 277 /* 278 * Another thread may have already initialized arenas[ind] if it's an 279 * auto arena. 280 */ 281 arena = arenas[ind]; 282 if (arena != NULL) { 283 assert(ind < narenas_auto); 284 return (arena); 285 } 286 287 /* Actually initialize the arena. */ 288 arena = arenas[ind] = arena_new(ind); 289 return (arena); 290} 291 292arena_t * 293arena_init(unsigned ind) 294{ 295 arena_t *arena; 296 297 malloc_mutex_lock(&arenas_lock); 298 arena = arena_init_locked(ind); 299 malloc_mutex_unlock(&arenas_lock); 300 return (arena); 301} 302 303unsigned 304narenas_total_get(void) 305{ 306 unsigned narenas; 307 308 malloc_mutex_lock(&arenas_lock); 309 narenas = narenas_total; 310 malloc_mutex_unlock(&arenas_lock); 311 312 return (narenas); 313} 314 315static void 316arena_bind_locked(tsd_t *tsd, unsigned ind) 317{ 318 arena_t *arena; 319 320 arena = arenas[ind]; 321 arena->nthreads++; 322 323 if (tsd_nominal(tsd)) 324 tsd_arena_set(tsd, arena); 325} 326 327static void 328arena_bind(tsd_t *tsd, unsigned ind) 329{ 330 331 malloc_mutex_lock(&arenas_lock); 332 arena_bind_locked(tsd, ind); 333 malloc_mutex_unlock(&arenas_lock); 334} 335 336void 337arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind) 338{ 339 arena_t *oldarena, *newarena; 340 341 malloc_mutex_lock(&arenas_lock); 342 oldarena = arenas[oldind]; 343 newarena = arenas[newind]; 344 oldarena->nthreads--; 345 newarena->nthreads++; 346 malloc_mutex_unlock(&arenas_lock); 347 tsd_arena_set(tsd, newarena); 348} 349 350unsigned 351arena_nbound(unsigned ind) 352{ 353 unsigned nthreads; 354 355 malloc_mutex_lock(&arenas_lock); 356 nthreads = arenas[ind]->nthreads; 357 malloc_mutex_unlock(&arenas_lock); 358 return (nthreads); 359} 360 361static void 362arena_unbind(tsd_t *tsd, unsigned ind) 363{ 364 arena_t *arena; 365 366 malloc_mutex_lock(&arenas_lock); 367 arena = arenas[ind]; 368 arena->nthreads--; 369 malloc_mutex_unlock(&arenas_lock); 370 tsd_arena_set(tsd, NULL); 371} 372 373arena_t * 374arena_get_hard(tsd_t *tsd, unsigned ind, bool init_if_missing) 375{ 376 arena_t *arena; 377 arena_t **arenas_cache = tsd_arenas_cache_get(tsd); 378 unsigned narenas_cache = tsd_narenas_cache_get(tsd); 379 unsigned narenas_actual = narenas_total_get(); 380 381 /* Deallocate old cache if it's too small. */ 382 if (arenas_cache != NULL && narenas_cache < narenas_actual) { 383 a0free(arenas_cache); 384 arenas_cache = NULL; 385 narenas_cache = 0; 386 tsd_arenas_cache_set(tsd, arenas_cache); 387 tsd_narenas_cache_set(tsd, narenas_cache); 388 } 389 390 /* Allocate cache if it's missing. */ 391 if (arenas_cache == NULL) { 392 bool *arenas_cache_bypassp = tsd_arenas_cache_bypassp_get(tsd); 393 assert(ind < narenas_actual || !init_if_missing); 394 narenas_cache = (ind < narenas_actual) ? narenas_actual : ind+1; 395 396 if (!*arenas_cache_bypassp) { 397 *arenas_cache_bypassp = true; 398 arenas_cache = (arena_t **)a0malloc(sizeof(arena_t *) * 399 narenas_cache); 400 *arenas_cache_bypassp = false; 401 } else 402 arenas_cache = NULL; 403 if (arenas_cache == NULL) { 404 /* 405 * This function must always tell the truth, even if 406 * it's slow, so don't let OOM or recursive allocation 407 * avoidance (note arenas_cache_bypass check) get in the 408 * way. 409 */ 410 if (ind >= narenas_actual) 411 return (NULL); 412 malloc_mutex_lock(&arenas_lock); 413 arena = arenas[ind]; 414 malloc_mutex_unlock(&arenas_lock); 415 return (arena); 416 } 417 tsd_arenas_cache_set(tsd, arenas_cache); 418 tsd_narenas_cache_set(tsd, narenas_cache); 419 } 420 421 /* 422 * Copy to cache. It's possible that the actual number of arenas has 423 * increased since narenas_total_get() was called above, but that causes 424 * no correctness issues unless two threads concurrently execute the 425 * arenas.extend mallctl, which we trust mallctl synchronization to 426 * prevent. 427 */ 428 malloc_mutex_lock(&arenas_lock); 429 memcpy(arenas_cache, arenas, sizeof(arena_t *) * narenas_actual); 430 malloc_mutex_unlock(&arenas_lock); 431 if (narenas_cache > narenas_actual) { 432 memset(&arenas_cache[narenas_actual], 0, sizeof(arena_t *) * 433 (narenas_cache - narenas_actual)); 434 } 435 436 /* Read the refreshed cache, and init the arena if necessary. */ 437 arena = arenas_cache[ind]; 438 if (init_if_missing && arena == NULL) 439 arena = arenas_cache[ind] = arena_init(ind); 440 return (arena); 441} 442 443/* Slow path, called only by arena_choose(). */ 444arena_t * 445arena_choose_hard(tsd_t *tsd) 446{ 447 arena_t *ret; 448 449 if (narenas_auto > 1) { 450 unsigned i, choose, first_null; 451 452 choose = 0; 453 first_null = narenas_auto; 454 malloc_mutex_lock(&arenas_lock); 455 assert(a0get() != NULL); 456 for (i = 1; i < narenas_auto; i++) { 457 if (arenas[i] != NULL) { 458 /* 459 * Choose the first arena that has the lowest 460 * number of threads assigned to it. 461 */ 462 if (arenas[i]->nthreads < 463 arenas[choose]->nthreads) 464 choose = i; 465 } else if (first_null == narenas_auto) { 466 /* 467 * Record the index of the first uninitialized 468 * arena, in case all extant arenas are in use. 469 * 470 * NB: It is possible for there to be 471 * discontinuities in terms of initialized 472 * versus uninitialized arenas, due to the 473 * "thread.arena" mallctl. 474 */ 475 first_null = i; 476 } 477 } 478 479 if (arenas[choose]->nthreads == 0 480 || first_null == narenas_auto) { 481 /* 482 * Use an unloaded arena, or the least loaded arena if 483 * all arenas are already initialized. 484 */ 485 ret = arenas[choose]; 486 } else { 487 /* Initialize a new arena. */ 488 choose = first_null; 489 ret = arena_init_locked(choose); 490 if (ret == NULL) { 491 malloc_mutex_unlock(&arenas_lock); 492 return (NULL); 493 } 494 } 495 arena_bind_locked(tsd, choose); 496 malloc_mutex_unlock(&arenas_lock); 497 } else { 498 ret = a0get(); 499 arena_bind(tsd, 0); 500 } 501 502 return (ret); 503} 504 505void 506thread_allocated_cleanup(tsd_t *tsd) 507{ 508 509 /* Do nothing. */ 510} 511 512void 513thread_deallocated_cleanup(tsd_t *tsd) 514{ 515 516 /* Do nothing. */ 517} 518 519void 520arena_cleanup(tsd_t *tsd) 521{ 522 arena_t *arena; 523 524 arena = tsd_arena_get(tsd); 525 if (arena != NULL) 526 arena_unbind(tsd, arena->ind); 527} 528 529void 530arenas_cache_cleanup(tsd_t *tsd) 531{ 532 arena_t **arenas_cache; 533 534 arenas_cache = tsd_arenas_cache_get(tsd); 535 if (arenas != NULL) 536 a0free(arenas_cache); 537} 538 539void 540narenas_cache_cleanup(tsd_t *tsd) 541{ 542 543 /* Do nothing. */ 544} 545 546void 547arenas_cache_bypass_cleanup(tsd_t *tsd) 548{ 549 550 /* Do nothing. */ 551} 552 553static void 554stats_print_atexit(void) 555{ 556 557 if (config_tcache && config_stats) { 558 unsigned narenas, i; 559 560 /* 561 * Merge stats from extant threads. This is racy, since 562 * individual threads do not lock when recording tcache stats 563 * events. As a consequence, the final stats may be slightly 564 * out of date by the time they are reported, if other threads 565 * continue to allocate. 566 */ 567 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { 568 arena_t *arena = arenas[i]; 569 if (arena != NULL) { 570 tcache_t *tcache; 571 572 /* 573 * tcache_stats_merge() locks bins, so if any 574 * code is introduced that acquires both arena 575 * and bin locks in the opposite order, 576 * deadlocks may result. 577 */ 578 malloc_mutex_lock(&arena->lock); 579 ql_foreach(tcache, &arena->tcache_ql, link) { 580 tcache_stats_merge(tcache, arena); 581 } 582 malloc_mutex_unlock(&arena->lock); 583 } 584 } 585 } 586 je_malloc_stats_print(NULL, NULL, NULL); 587} 588 589/* 590 * End miscellaneous support functions. 591 */ 592/******************************************************************************/ 593/* 594 * Begin initialization functions. 595 */ 596 597static unsigned 598malloc_ncpus(void) 599{ 600 long result; 601 602#ifdef _WIN32 603 SYSTEM_INFO si; 604 GetSystemInfo(&si); 605 result = si.dwNumberOfProcessors; 606#else 607 result = sysconf(_SC_NPROCESSORS_ONLN); 608#endif 609 return ((result == -1) ? 1 : (unsigned)result); 610} 611 612static bool 613malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, 614 char const **v_p, size_t *vlen_p) 615{ 616 bool accept; 617 const char *opts = *opts_p; 618 619 *k_p = opts; 620 621 for (accept = false; !accept;) { 622 switch (*opts) { 623 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': 624 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': 625 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': 626 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': 627 case 'Y': case 'Z': 628 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': 629 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': 630 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': 631 case 's': case 't': case 'u': case 'v': case 'w': case 'x': 632 case 'y': case 'z': 633 case '0': case '1': case '2': case '3': case '4': case '5': 634 case '6': case '7': case '8': case '9': 635 case '_': 636 opts++; 637 break; 638 case ':': 639 opts++; 640 *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p; 641 *v_p = opts; 642 accept = true; 643 break; 644 case '\0': 645 if (opts != *opts_p) { 646 malloc_write("<jemalloc>: Conf string ends " 647 "with key\n"); 648 } 649 return (true); 650 default: 651 malloc_write("<jemalloc>: Malformed conf string\n"); 652 return (true); 653 } 654 } 655 656 for (accept = false; !accept;) { 657 switch (*opts) { 658 case ',': 659 opts++; 660 /* 661 * Look ahead one character here, because the next time 662 * this function is called, it will assume that end of 663 * input has been cleanly reached if no input remains, 664 * but we have optimistically already consumed the 665 * comma if one exists. 666 */ 667 if (*opts == '\0') { 668 malloc_write("<jemalloc>: Conf string ends " 669 "with comma\n"); 670 } 671 *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p; 672 accept = true; 673 break; 674 case '\0': 675 *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p; 676 accept = true; 677 break; 678 default: 679 opts++; 680 break; 681 } 682 } 683 684 *opts_p = opts; 685 return (false); 686} 687 688static void 689malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, 690 size_t vlen) 691{ 692 693 malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k, 694 (int)vlen, v); 695} 696 697static void 698malloc_conf_init(void) 699{ 700 unsigned i; 701 char buf[PATH_MAX + 1]; 702 const char *opts, *k, *v; 703 size_t klen, vlen; 704 705 /* 706 * Automatically configure valgrind before processing options. The 707 * valgrind option remains in jemalloc 3.x for compatibility reasons. 708 */ 709 if (config_valgrind) { 710 in_valgrind = (RUNNING_ON_VALGRIND != 0) ? true : false; 711 if (config_fill && unlikely(in_valgrind)) { 712 opt_junk = false; 713 assert(!opt_zero); 714 opt_quarantine = JEMALLOC_VALGRIND_QUARANTINE_DEFAULT; 715 opt_redzone = true; 716 } 717 if (config_tcache && unlikely(in_valgrind)) 718 opt_tcache = false; 719 } 720 721 for (i = 0; i < 3; i++) { 722 /* Get runtime configuration. */ 723 switch (i) { 724 case 0: 725 if (je_malloc_conf != NULL) { 726 /* 727 * Use options that were compiled into the 728 * program. 729 */ 730 opts = je_malloc_conf; 731 } else { 732 /* No configuration specified. */ 733 buf[0] = '\0'; 734 opts = buf; 735 } 736 break; 737 case 1: { 738 int linklen = 0; 739#ifndef _WIN32 740 int saved_errno = errno; 741 const char *linkname = 742# ifdef JEMALLOC_PREFIX 743 "/etc/"JEMALLOC_PREFIX"malloc.conf" 744# else 745 "/etc/malloc.conf" 746# endif 747 ; 748 749 /* 750 * Try to use the contents of the "/etc/malloc.conf" 751 * symbolic link's name. 752 */ 753 linklen = readlink(linkname, buf, sizeof(buf) - 1); 754 if (linklen == -1) { 755 /* No configuration specified. */ 756 linklen = 0; 757 /* restore errno */ 758 set_errno(saved_errno); 759 } 760#endif 761 buf[linklen] = '\0'; 762 opts = buf; 763 break; 764 } case 2: { 765 const char *envname = 766#ifdef JEMALLOC_PREFIX 767 JEMALLOC_CPREFIX"MALLOC_CONF" 768#else 769 "MALLOC_CONF" 770#endif 771 ; 772 773 if ((opts = getenv(envname)) != NULL) { 774 /* 775 * Do nothing; opts is already initialized to 776 * the value of the MALLOC_CONF environment 777 * variable. 778 */ 779 } else { 780 /* No configuration specified. */ 781 buf[0] = '\0'; 782 opts = buf; 783 } 784 break; 785 } default: 786 not_reached(); 787 buf[0] = '\0'; 788 opts = buf; 789 } 790 791 while (*opts != '\0' && !malloc_conf_next(&opts, &k, &klen, &v, 792 &vlen)) { 793#define CONF_MATCH(n) \ 794 (sizeof(n)-1 == klen && strncmp(n, k, klen) == 0) 795#define CONF_HANDLE_BOOL(o, n, cont) \ 796 if (CONF_MATCH(n)) { \ 797 if (strncmp("true", v, vlen) == 0 && \ 798 vlen == sizeof("true")-1) \ 799 o = true; \ 800 else if (strncmp("false", v, vlen) == \ 801 0 && vlen == sizeof("false")-1) \ 802 o = false; \ 803 else { \ 804 malloc_conf_error( \ 805 "Invalid conf value", \ 806 k, klen, v, vlen); \ 807 } \ 808 if (cont) \ 809 continue; \ 810 } 811#define CONF_HANDLE_SIZE_T(o, n, min, max, clip) \ 812 if (CONF_MATCH(n)) { \ 813 uintmax_t um; \ 814 char *end; \ 815 \ 816 set_errno(0); \ 817 um = malloc_strtoumax(v, &end, 0); \ 818 if (get_errno() != 0 || (uintptr_t)end -\ 819 (uintptr_t)v != vlen) { \ 820 malloc_conf_error( \ 821 "Invalid conf value", \ 822 k, klen, v, vlen); \ 823 } else if (clip) { \ 824 if ((min) != 0 && um < (min)) \ 825 o = (min); \ 826 else if (um > (max)) \ 827 o = (max); \ 828 else \ 829 o = um; \ 830 } else { \ 831 if (((min) != 0 && um < (min)) \ 832 || um > (max)) { \ 833 malloc_conf_error( \ 834 "Out-of-range " \ 835 "conf value", \ 836 k, klen, v, vlen); \ 837 } else \ 838 o = um; \ 839 } \ 840 continue; \ 841 } 842#define CONF_HANDLE_SSIZE_T(o, n, min, max) \ 843 if (CONF_MATCH(n)) { \ 844 long l; \ 845 char *end; \ 846 \ 847 set_errno(0); \ 848 l = strtol(v, &end, 0); \ 849 if (get_errno() != 0 || (uintptr_t)end -\ 850 (uintptr_t)v != vlen) { \ 851 malloc_conf_error( \ 852 "Invalid conf value", \ 853 k, klen, v, vlen); \ 854 } else if (l < (ssize_t)(min) || l > \ 855 (ssize_t)(max)) { \ 856 malloc_conf_error( \ 857 "Out-of-range conf value", \ 858 k, klen, v, vlen); \ 859 } else \ 860 o = l; \ 861 continue; \ 862 } 863#define CONF_HANDLE_CHAR_P(o, n, d) \ 864 if (CONF_MATCH(n)) { \ 865 size_t cpylen = (vlen <= \ 866 sizeof(o)-1) ? vlen : \ 867 sizeof(o)-1; \ 868 strncpy(o, v, cpylen); \ 869 o[cpylen] = '\0'; \ 870 continue; \ 871 } 872 873 CONF_HANDLE_BOOL(opt_abort, "abort", true) 874 /* 875 * Chunks always require at least one header page, 876 * as many as 2^(LG_SIZE_CLASS_GROUP+1) data pages, and 877 * possibly an additional page in the presence of 878 * redzones. In order to simplify options processing, 879 * use a conservative bound that accommodates all these 880 * constraints. 881 */ 882 CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE + 883 LG_SIZE_CLASS_GROUP + (config_fill ? 2 : 1), 884 (sizeof(size_t) << 3) - 1, true) 885 if (strncmp("dss", k, klen) == 0) { 886 int i; 887 bool match = false; 888 for (i = 0; i < dss_prec_limit; i++) { 889 if (strncmp(dss_prec_names[i], v, vlen) 890 == 0) { 891 if (chunk_dss_prec_set(i)) { 892 malloc_conf_error( 893 "Error setting dss", 894 k, klen, v, vlen); 895 } else { 896 opt_dss = 897 dss_prec_names[i]; 898 match = true; 899 break; 900 } 901 } 902 } 903 if (!match) { 904 malloc_conf_error("Invalid conf value", 905 k, klen, v, vlen); 906 } 907 continue; 908 } 909 CONF_HANDLE_SIZE_T(opt_narenas, "narenas", 1, 910 SIZE_T_MAX, false) 911 CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult", 912 -1, (sizeof(size_t) << 3) - 1) 913 CONF_HANDLE_BOOL(opt_stats_print, "stats_print", true) 914 if (config_fill) { 915 CONF_HANDLE_BOOL(opt_junk, "junk", true) 916 CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine", 917 0, SIZE_T_MAX, false) 918 CONF_HANDLE_BOOL(opt_redzone, "redzone", true) 919 CONF_HANDLE_BOOL(opt_zero, "zero", true) 920 } 921 if (config_utrace) { 922 CONF_HANDLE_BOOL(opt_utrace, "utrace", true) 923 } 924 if (config_xmalloc) { 925 CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc", true) 926 } 927 if (config_tcache) { 928 CONF_HANDLE_BOOL(opt_tcache, "tcache", 929 !config_valgrind || !in_valgrind) 930 if (CONF_MATCH("tcache")) { 931 assert(config_valgrind && in_valgrind); 932 if (opt_tcache) { 933 opt_tcache = false; 934 malloc_conf_error( 935 "tcache cannot be enabled " 936 "while running inside Valgrind", 937 k, klen, v, vlen); 938 } 939 continue; 940 } 941 CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, 942 "lg_tcache_max", -1, 943 (sizeof(size_t) << 3) - 1) 944 } 945 if (config_prof) { 946 CONF_HANDLE_BOOL(opt_prof, "prof", true) 947 CONF_HANDLE_CHAR_P(opt_prof_prefix, 948 "prof_prefix", "jeprof") 949 CONF_HANDLE_BOOL(opt_prof_active, "prof_active", 950 true) 951 CONF_HANDLE_BOOL(opt_prof_thread_active_init, 952 "prof_thread_active_init", true) 953 CONF_HANDLE_SIZE_T(opt_lg_prof_sample, 954 "lg_prof_sample", 0, 955 (sizeof(uint64_t) << 3) - 1, true) 956 CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum", 957 true) 958 CONF_HANDLE_SSIZE_T(opt_lg_prof_interval, 959 "lg_prof_interval", -1, 960 (sizeof(uint64_t) << 3) - 1) 961 CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump", 962 true) 963 CONF_HANDLE_BOOL(opt_prof_final, "prof_final", 964 true) 965 CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak", 966 true) 967 } 968 malloc_conf_error("Invalid conf pair", k, klen, v, 969 vlen); 970#undef CONF_MATCH 971#undef CONF_HANDLE_BOOL 972#undef CONF_HANDLE_SIZE_T 973#undef CONF_HANDLE_SSIZE_T 974#undef CONF_HANDLE_CHAR_P 975 } 976 } 977} 978 979static bool 980malloc_init_hard(void) 981{ 982 arena_t *init_arenas[1]; 983 984 malloc_mutex_lock(&init_lock); 985 if (malloc_initialized || IS_INITIALIZER) { 986 /* 987 * Another thread initialized the allocator before this one 988 * acquired init_lock, or this thread is the initializing 989 * thread, and it is recursively allocating. 990 */ 991 malloc_mutex_unlock(&init_lock); 992 return (false); 993 } 994#ifdef JEMALLOC_THREADED_INIT 995 if (malloc_initializer != NO_INITIALIZER && !IS_INITIALIZER) { 996 /* Busy-wait until the initializing thread completes. */ 997 do { 998 malloc_mutex_unlock(&init_lock); 999 CPU_SPINWAIT; 1000 malloc_mutex_lock(&init_lock); 1001 } while (!malloc_initialized); 1002 malloc_mutex_unlock(&init_lock); 1003 return (false); 1004 } 1005#endif 1006 malloc_initializer = INITIALIZER; 1007 1008 if (malloc_tsd_boot0()) { 1009 malloc_mutex_unlock(&init_lock); 1010 return (true); 1011 } 1012 1013 if (config_prof) 1014 prof_boot0(); 1015 1016 malloc_conf_init(); 1017 1018 if (opt_stats_print) { 1019 /* Print statistics at exit. */ 1020 if (atexit(stats_print_atexit) != 0) { 1021 malloc_write("<jemalloc>: Error in atexit()\n"); 1022 if (opt_abort) 1023 abort(); 1024 } 1025 } 1026 1027 if (base_boot()) { 1028 malloc_mutex_unlock(&init_lock); 1029 return (true); 1030 } 1031 1032 if (chunk_boot()) { 1033 malloc_mutex_unlock(&init_lock); 1034 return (true); 1035 } 1036 1037 if (ctl_boot()) { 1038 malloc_mutex_unlock(&init_lock); 1039 return (true); 1040 } 1041 1042 if (config_prof) 1043 prof_boot1(); 1044 1045 arena_boot(); 1046 1047 if (config_tcache && tcache_boot()) { 1048 malloc_mutex_unlock(&init_lock); 1049 return (true); 1050 } 1051 1052 if (huge_boot()) { 1053 malloc_mutex_unlock(&init_lock); 1054 return (true); 1055 } 1056 1057 if (malloc_mutex_init(&arenas_lock)) { 1058 malloc_mutex_unlock(&init_lock); 1059 return (true); 1060 } 1061 1062 /* 1063 * Create enough scaffolding to allow recursive allocation in 1064 * malloc_ncpus(). 1065 */ 1066 narenas_total = narenas_auto = 1; 1067 arenas = init_arenas; 1068 memset(arenas, 0, sizeof(arena_t *) * narenas_auto); 1069 1070 /* 1071 * Initialize one arena here. The rest are lazily created in 1072 * arena_choose_hard(). 1073 */ 1074 a0 = arena_init(0); 1075 if (a0 == NULL) { 1076 malloc_mutex_unlock(&init_lock); 1077 return (true); 1078 } 1079 1080 if (config_prof && prof_boot2()) { 1081 malloc_mutex_unlock(&init_lock); 1082 return (true); 1083 } 1084 1085 malloc_mutex_unlock(&init_lock); 1086 /**********************************************************************/ 1087 /* Recursive allocation may follow. */ 1088 1089 ncpus = malloc_ncpus(); 1090 1091#if (!defined(JEMALLOC_MUTEX_INIT_CB) && !defined(JEMALLOC_ZONE) \ 1092 && !defined(_WIN32) && !defined(__native_client__)) 1093 /* LinuxThreads's pthread_atfork() allocates. */ 1094 if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent, 1095 jemalloc_postfork_child) != 0) { 1096 malloc_write("<jemalloc>: Error in pthread_atfork()\n"); 1097 if (opt_abort) 1098 abort(); 1099 } 1100#endif 1101 1102 /* Done recursively allocating. */ 1103 /**********************************************************************/ 1104 malloc_mutex_lock(&init_lock); 1105 1106 if (mutex_boot()) { 1107 malloc_mutex_unlock(&init_lock); 1108 return (true); 1109 } 1110 1111 if (opt_narenas == 0) { 1112 /* 1113 * For SMP systems, create more than one arena per CPU by 1114 * default. 1115 */ 1116 if (ncpus > 1) 1117 opt_narenas = ncpus << 2; 1118 else 1119 opt_narenas = 1; 1120 } 1121 narenas_auto = opt_narenas; 1122 /* 1123 * Make sure that the arenas array can be allocated. In practice, this 1124 * limit is enough to allow the allocator to function, but the ctl 1125 * machinery will fail to allocate memory at far lower limits. 1126 */ 1127 if (narenas_auto > chunksize / sizeof(arena_t *)) { 1128 narenas_auto = chunksize / sizeof(arena_t *); 1129 malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n", 1130 narenas_auto); 1131 } 1132 narenas_total = narenas_auto; 1133 1134 /* Allocate and initialize arenas. */ 1135 arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas_total); 1136 if (arenas == NULL) { 1137 malloc_mutex_unlock(&init_lock); 1138 return (true); 1139 } 1140 /* 1141 * Zero the array. In practice, this should always be pre-zeroed, 1142 * since it was just mmap()ed, but let's be sure. 1143 */ 1144 memset(arenas, 0, sizeof(arena_t *) * narenas_total); 1145 /* Copy the pointer to the one arena that was already initialized. */ 1146 arenas[0] = init_arenas[0]; 1147 1148 malloc_initialized = true; 1149 malloc_mutex_unlock(&init_lock); 1150 malloc_tsd_boot1(); 1151 1152 return (false); 1153} 1154 1155/* 1156 * End initialization functions. 1157 */ 1158/******************************************************************************/ 1159/* 1160 * Begin malloc(3)-compatible functions. 1161 */ 1162 1163static void * 1164imalloc_prof_sample(tsd_t *tsd, size_t usize, prof_tctx_t *tctx) 1165{ 1166 void *p; 1167 1168 if (tctx == NULL) 1169 return (NULL); 1170 if (usize <= SMALL_MAXCLASS) { 1171 p = imalloc(tsd, LARGE_MINCLASS); 1172 if (p == NULL) 1173 return (NULL); 1174 arena_prof_promoted(p, usize); 1175 } else 1176 p = imalloc(tsd, usize); 1177 1178 return (p); 1179} 1180 1181JEMALLOC_ALWAYS_INLINE_C void * 1182imalloc_prof(tsd_t *tsd, size_t usize) 1183{ 1184 void *p; 1185 prof_tctx_t *tctx; 1186 1187 tctx = prof_alloc_prep(tsd, usize, true); 1188 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) 1189 p = imalloc_prof_sample(tsd, usize, tctx); 1190 else 1191 p = imalloc(tsd, usize); 1192 if (p == NULL) { 1193 prof_alloc_rollback(tsd, tctx, true); 1194 return (NULL); 1195 } 1196 prof_malloc(p, usize, tctx); 1197 1198 return (p); 1199} 1200 1201JEMALLOC_ALWAYS_INLINE_C void * 1202imalloc_body(size_t size, tsd_t **tsd, size_t *usize) 1203{ 1204 1205 if (unlikely(malloc_init())) 1206 return (NULL); 1207 *tsd = tsd_fetch(); 1208 1209 if (config_prof && opt_prof) { 1210 *usize = s2u(size); 1211 return (imalloc_prof(*tsd, *usize)); 1212 } 1213 1214 if (config_stats || (config_valgrind && unlikely(in_valgrind))) 1215 *usize = s2u(size); 1216 return (imalloc(*tsd, size)); 1217} 1218 1219void * 1220je_malloc(size_t size) 1221{ 1222 void *ret; 1223 tsd_t *tsd; 1224 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1225 1226 if (size == 0) 1227 size = 1; 1228 1229 ret = imalloc_body(size, &tsd, &usize); 1230 if (unlikely(ret == NULL)) { 1231 if (config_xmalloc && unlikely(opt_xmalloc)) { 1232 malloc_write("<jemalloc>: Error in malloc(): " 1233 "out of memory\n"); 1234 abort(); 1235 } 1236 set_errno(ENOMEM); 1237 } 1238 if (config_stats && likely(ret != NULL)) { 1239 assert(usize == isalloc(ret, config_prof)); 1240 *tsd_thread_allocatedp_get(tsd) += usize; 1241 } 1242 UTRACE(0, size, ret); 1243 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, false); 1244 return (ret); 1245} 1246 1247static void * 1248imemalign_prof_sample(tsd_t *tsd, size_t alignment, size_t usize, 1249 prof_tctx_t *tctx) 1250{ 1251 void *p; 1252 1253 if (tctx == NULL) 1254 return (NULL); 1255 if (usize <= SMALL_MAXCLASS) { 1256 assert(sa2u(LARGE_MINCLASS, alignment) == LARGE_MINCLASS); 1257 p = imalloc(tsd, LARGE_MINCLASS); 1258 if (p == NULL) 1259 return (NULL); 1260 arena_prof_promoted(p, usize); 1261 } else 1262 p = ipalloc(tsd, usize, alignment, false); 1263 1264 return (p); 1265} 1266 1267JEMALLOC_ALWAYS_INLINE_C void * 1268imemalign_prof(tsd_t *tsd, size_t alignment, size_t usize) 1269{ 1270 void *p; 1271 prof_tctx_t *tctx; 1272 1273 tctx = prof_alloc_prep(tsd, usize, true); 1274 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) 1275 p = imemalign_prof_sample(tsd, alignment, usize, tctx); 1276 else 1277 p = ipalloc(tsd, usize, alignment, false); 1278 if (p == NULL) { 1279 prof_alloc_rollback(tsd, tctx, true); 1280 return (NULL); 1281 } 1282 prof_malloc(p, usize, tctx); 1283 1284 return (p); 1285} 1286 1287JEMALLOC_ATTR(nonnull(1)) 1288static int 1289imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment) 1290{ 1291 int ret; 1292 tsd_t *tsd; 1293 size_t usize; 1294 void *result; 1295 1296 assert(min_alignment != 0); 1297 1298 if (unlikely(malloc_init())) { 1299 result = NULL; 1300 goto label_oom; 1301 } else { 1302 tsd = tsd_fetch(); 1303 if (size == 0) 1304 size = 1; 1305 1306 /* Make sure that alignment is a large enough power of 2. */ 1307 if (unlikely(((alignment - 1) & alignment) != 0 1308 || (alignment < min_alignment))) { 1309 if (config_xmalloc && unlikely(opt_xmalloc)) { 1310 malloc_write("<jemalloc>: Error allocating " 1311 "aligned memory: invalid alignment\n"); 1312 abort(); 1313 } 1314 result = NULL; 1315 ret = EINVAL; 1316 goto label_return; 1317 } 1318 1319 usize = sa2u(size, alignment); 1320 if (unlikely(usize == 0)) { 1321 result = NULL; 1322 goto label_oom; 1323 } 1324 1325 if (config_prof && opt_prof) 1326 result = imemalign_prof(tsd, alignment, usize); 1327 else 1328 result = ipalloc(tsd, usize, alignment, false); 1329 if (unlikely(result == NULL)) 1330 goto label_oom; 1331 } 1332 1333 *memptr = result; 1334 ret = 0; 1335label_return: 1336 if (config_stats && likely(result != NULL)) { 1337 assert(usize == isalloc(result, config_prof)); 1338 *tsd_thread_allocatedp_get(tsd) += usize; 1339 } 1340 UTRACE(0, size, result); 1341 return (ret); 1342label_oom: 1343 assert(result == NULL); 1344 if (config_xmalloc && unlikely(opt_xmalloc)) { 1345 malloc_write("<jemalloc>: Error allocating aligned memory: " 1346 "out of memory\n"); 1347 abort(); 1348 } 1349 ret = ENOMEM; 1350 goto label_return; 1351} 1352 1353int 1354je_posix_memalign(void **memptr, size_t alignment, size_t size) 1355{ 1356 int ret = imemalign(memptr, alignment, size, sizeof(void *)); 1357 JEMALLOC_VALGRIND_MALLOC(ret == 0, *memptr, isalloc(*memptr, 1358 config_prof), false); 1359 return (ret); 1360} 1361 1362void * 1363je_aligned_alloc(size_t alignment, size_t size) 1364{ 1365 void *ret; 1366 int err; 1367 1368 if (unlikely((err = imemalign(&ret, alignment, size, 1)) != 0)) { 1369 ret = NULL; 1370 set_errno(err); 1371 } 1372 JEMALLOC_VALGRIND_MALLOC(err == 0, ret, isalloc(ret, config_prof), 1373 false); 1374 return (ret); 1375} 1376 1377static void * 1378icalloc_prof_sample(tsd_t *tsd, size_t usize, prof_tctx_t *tctx) 1379{ 1380 void *p; 1381 1382 if (tctx == NULL) 1383 return (NULL); 1384 if (usize <= SMALL_MAXCLASS) { 1385 p = icalloc(tsd, LARGE_MINCLASS); 1386 if (p == NULL) 1387 return (NULL); 1388 arena_prof_promoted(p, usize); 1389 } else 1390 p = icalloc(tsd, usize); 1391 1392 return (p); 1393} 1394 1395JEMALLOC_ALWAYS_INLINE_C void * 1396icalloc_prof(tsd_t *tsd, size_t usize) 1397{ 1398 void *p; 1399 prof_tctx_t *tctx; 1400 1401 tctx = prof_alloc_prep(tsd, usize, true); 1402 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) 1403 p = icalloc_prof_sample(tsd, usize, tctx); 1404 else 1405 p = icalloc(tsd, usize); 1406 if (p == NULL) { 1407 prof_alloc_rollback(tsd, tctx, true); 1408 return (NULL); 1409 } 1410 prof_malloc(p, usize, tctx); 1411 1412 return (p); 1413} 1414 1415void * 1416je_calloc(size_t num, size_t size) 1417{ 1418 void *ret; 1419 tsd_t *tsd; 1420 size_t num_size; 1421 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1422 1423 if (unlikely(malloc_init())) { 1424 num_size = 0; 1425 ret = NULL; 1426 goto label_return; 1427 } 1428 tsd = tsd_fetch(); 1429 1430 num_size = num * size; 1431 if (unlikely(num_size == 0)) { 1432 if (num == 0 || size == 0) 1433 num_size = 1; 1434 else { 1435 ret = NULL; 1436 goto label_return; 1437 } 1438 /* 1439 * Try to avoid division here. We know that it isn't possible to 1440 * overflow during multiplication if neither operand uses any of the 1441 * most significant half of the bits in a size_t. 1442 */ 1443 } else if (unlikely(((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 1444 2))) && (num_size / size != num))) { 1445 /* size_t overflow. */ 1446 ret = NULL; 1447 goto label_return; 1448 } 1449 1450 if (config_prof && opt_prof) { 1451 usize = s2u(num_size); 1452 ret = icalloc_prof(tsd, usize); 1453 } else { 1454 if (config_stats || (config_valgrind && unlikely(in_valgrind))) 1455 usize = s2u(num_size); 1456 ret = icalloc(tsd, num_size); 1457 } 1458 1459label_return: 1460 if (unlikely(ret == NULL)) { 1461 if (config_xmalloc && unlikely(opt_xmalloc)) { 1462 malloc_write("<jemalloc>: Error in calloc(): out of " 1463 "memory\n"); 1464 abort(); 1465 } 1466 set_errno(ENOMEM); 1467 } 1468 if (config_stats && likely(ret != NULL)) { 1469 assert(usize == isalloc(ret, config_prof)); 1470 *tsd_thread_allocatedp_get(tsd) += usize; 1471 } 1472 UTRACE(0, num_size, ret); 1473 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, true); 1474 return (ret); 1475} 1476 1477static void * 1478irealloc_prof_sample(tsd_t *tsd, void *oldptr, size_t usize, prof_tctx_t *tctx) 1479{ 1480 void *p; 1481 1482 if (tctx == NULL) 1483 return (NULL); 1484 if (usize <= SMALL_MAXCLASS) { 1485 p = iralloc(tsd, oldptr, LARGE_MINCLASS, 0, false); 1486 if (p == NULL) 1487 return (NULL); 1488 arena_prof_promoted(p, usize); 1489 } else 1490 p = iralloc(tsd, oldptr, usize, 0, false); 1491 1492 return (p); 1493} 1494 1495JEMALLOC_ALWAYS_INLINE_C void * 1496irealloc_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t usize) 1497{ 1498 void *p; 1499 prof_tctx_t *old_tctx, *tctx; 1500 1501 old_tctx = prof_tctx_get(oldptr); 1502 tctx = prof_alloc_prep(tsd, usize, true); 1503 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) 1504 p = irealloc_prof_sample(tsd, oldptr, usize, tctx); 1505 else 1506 p = iralloc(tsd, oldptr, usize, 0, false); 1507 if (p == NULL) 1508 return (NULL); 1509 prof_realloc(tsd, p, usize, tctx, true, old_usize, old_tctx); 1510 1511 return (p); 1512} 1513 1514JEMALLOC_INLINE_C void 1515ifree(tsd_t *tsd, void *ptr, bool try_tcache) 1516{ 1517 size_t usize; 1518 UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); 1519 1520 assert(ptr != NULL); 1521 assert(malloc_initialized || IS_INITIALIZER); 1522 1523 if (config_prof && opt_prof) { 1524 usize = isalloc(ptr, config_prof); 1525 prof_free(tsd, ptr, usize); 1526 } else if (config_stats || config_valgrind) 1527 usize = isalloc(ptr, config_prof); 1528 if (config_stats) 1529 *tsd_thread_deallocatedp_get(tsd) += usize; 1530 if (config_valgrind && unlikely(in_valgrind)) 1531 rzsize = p2rz(ptr); 1532 iqalloc(tsd, ptr, try_tcache); 1533 JEMALLOC_VALGRIND_FREE(ptr, rzsize); 1534} 1535 1536JEMALLOC_INLINE_C void 1537isfree(tsd_t *tsd, void *ptr, size_t usize, bool try_tcache) 1538{ 1539 UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); 1540 1541 assert(ptr != NULL); 1542 assert(malloc_initialized || IS_INITIALIZER); 1543 1544 if (config_prof && opt_prof) 1545 prof_free(tsd, ptr, usize); 1546 if (config_stats) 1547 *tsd_thread_deallocatedp_get(tsd) += usize; 1548 if (config_valgrind && unlikely(in_valgrind)) 1549 rzsize = p2rz(ptr); 1550 isqalloc(tsd, ptr, usize, try_tcache); 1551 JEMALLOC_VALGRIND_FREE(ptr, rzsize); 1552} 1553 1554void * 1555je_realloc(void *ptr, size_t size) 1556{ 1557 void *ret; 1558 tsd_t *tsd JEMALLOC_CC_SILENCE_INIT(NULL); 1559 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1560 size_t old_usize = 0; 1561 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 1562 1563 if (unlikely(size == 0)) { 1564 if (ptr != NULL) { 1565 /* realloc(ptr, 0) is equivalent to free(ptr). */ 1566 UTRACE(ptr, 0, 0); 1567 tsd = tsd_fetch(); 1568 ifree(tsd, ptr, true); 1569 return (NULL); 1570 } 1571 size = 1; 1572 } 1573 1574 if (likely(ptr != NULL)) { 1575 assert(malloc_initialized || IS_INITIALIZER); 1576 malloc_thread_init(); 1577 tsd = tsd_fetch(); 1578 1579 if ((config_prof && opt_prof) || config_stats || 1580 (config_valgrind && unlikely(in_valgrind))) 1581 old_usize = isalloc(ptr, config_prof); 1582 if (config_valgrind && unlikely(in_valgrind)) 1583 old_rzsize = config_prof ? p2rz(ptr) : u2rz(old_usize); 1584 1585 if (config_prof && opt_prof) { 1586 usize = s2u(size); 1587 ret = irealloc_prof(tsd, ptr, old_usize, usize); 1588 } else { 1589 if (config_stats || (config_valgrind && 1590 unlikely(in_valgrind))) 1591 usize = s2u(size); 1592 ret = iralloc(tsd, ptr, size, 0, false); 1593 } 1594 } else { 1595 /* realloc(NULL, size) is equivalent to malloc(size). */ 1596 ret = imalloc_body(size, &tsd, &usize); 1597 } 1598 1599 if (unlikely(ret == NULL)) { 1600 if (config_xmalloc && unlikely(opt_xmalloc)) { 1601 malloc_write("<jemalloc>: Error in realloc(): " 1602 "out of memory\n"); 1603 abort(); 1604 } 1605 set_errno(ENOMEM); 1606 } 1607 if (config_stats && likely(ret != NULL)) { 1608 assert(usize == isalloc(ret, config_prof)); 1609 *tsd_thread_allocatedp_get(tsd) += usize; 1610 *tsd_thread_deallocatedp_get(tsd) += old_usize; 1611 } 1612 UTRACE(ptr, size, ret); 1613 JEMALLOC_VALGRIND_REALLOC(true, ret, usize, true, ptr, old_usize, 1614 old_rzsize, true, false); 1615 return (ret); 1616} 1617 1618void 1619je_free(void *ptr) 1620{ 1621 1622 UTRACE(ptr, 0, 0); 1623 if (likely(ptr != NULL)) 1624 ifree(tsd_fetch(), ptr, true); 1625} 1626 1627/* 1628 * End malloc(3)-compatible functions. 1629 */ 1630/******************************************************************************/ 1631/* 1632 * Begin non-standard override functions. 1633 */ 1634 1635#ifdef JEMALLOC_OVERRIDE_MEMALIGN 1636void * 1637je_memalign(size_t alignment, size_t size) 1638{ 1639 void *ret JEMALLOC_CC_SILENCE_INIT(NULL); 1640 imemalign(&ret, alignment, size, 1); 1641 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); 1642 return (ret); 1643} 1644#endif 1645 1646#ifdef JEMALLOC_OVERRIDE_VALLOC 1647void * 1648je_valloc(size_t size) 1649{ 1650 void *ret JEMALLOC_CC_SILENCE_INIT(NULL); 1651 imemalign(&ret, PAGE, size, 1); 1652 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); 1653 return (ret); 1654} 1655#endif 1656 1657/* 1658 * is_malloc(je_malloc) is some macro magic to detect if jemalloc_defs.h has 1659 * #define je_malloc malloc 1660 */ 1661#define malloc_is_malloc 1 1662#define is_malloc_(a) malloc_is_ ## a 1663#define is_malloc(a) is_malloc_(a) 1664 1665#if ((is_malloc(je_malloc) == 1) && defined(JEMALLOC_GLIBC_MALLOC_HOOK)) 1666/* 1667 * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible 1668 * to inconsistently reference libc's malloc(3)-compatible functions 1669 * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541). 1670 * 1671 * These definitions interpose hooks in glibc. The functions are actually 1672 * passed an extra argument for the caller return address, which will be 1673 * ignored. 1674 */ 1675JEMALLOC_EXPORT void (*__free_hook)(void *ptr) = je_free; 1676JEMALLOC_EXPORT void *(*__malloc_hook)(size_t size) = je_malloc; 1677JEMALLOC_EXPORT void *(*__realloc_hook)(void *ptr, size_t size) = je_realloc; 1678# ifdef JEMALLOC_GLIBC_MEMALIGN_HOOK 1679JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) = 1680 je_memalign; 1681# endif 1682#endif 1683 1684/* 1685 * End non-standard override functions. 1686 */ 1687/******************************************************************************/ 1688/* 1689 * Begin non-standard functions. 1690 */ 1691 1692JEMALLOC_ALWAYS_INLINE_C bool 1693imallocx_flags_decode_hard(tsd_t *tsd, size_t size, int flags, size_t *usize, 1694 size_t *alignment, bool *zero, bool *try_tcache, arena_t **arena) 1695{ 1696 1697 if ((flags & MALLOCX_LG_ALIGN_MASK) == 0) { 1698 *alignment = 0; 1699 *usize = s2u(size); 1700 } else { 1701 *alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags); 1702 *usize = sa2u(size, *alignment); 1703 } 1704 *zero = MALLOCX_ZERO_GET(flags); 1705 if ((flags & MALLOCX_ARENA_MASK) != 0) { 1706 unsigned arena_ind = MALLOCX_ARENA_GET(flags); 1707 *try_tcache = false; 1708 *arena = arena_get(tsd, arena_ind, true, true); 1709 if (unlikely(*arena == NULL)) 1710 return (true); 1711 } else { 1712 *try_tcache = true; 1713 *arena = NULL; 1714 } 1715 return (false); 1716} 1717 1718JEMALLOC_ALWAYS_INLINE_C bool 1719imallocx_flags_decode(tsd_t *tsd, size_t size, int flags, size_t *usize, 1720 size_t *alignment, bool *zero, bool *try_tcache, arena_t **arena) 1721{ 1722 1723 if (likely(flags == 0)) { 1724 *usize = s2u(size); 1725 assert(usize != 0); 1726 *alignment = 0; 1727 *zero = false; 1728 *try_tcache = true; 1729 *arena = NULL; 1730 return (false); 1731 } else { 1732 return (imallocx_flags_decode_hard(tsd, size, flags, usize, 1733 alignment, zero, try_tcache, arena)); 1734 } 1735} 1736 1737JEMALLOC_ALWAYS_INLINE_C void * 1738imallocx_flags(tsd_t *tsd, size_t usize, size_t alignment, bool zero, 1739 bool try_tcache, arena_t *arena) 1740{ 1741 1742 if (alignment != 0) { 1743 return (ipalloct(tsd, usize, alignment, zero, try_tcache, 1744 arena)); 1745 } 1746 if (zero) 1747 return (icalloct(tsd, usize, try_tcache, arena)); 1748 return (imalloct(tsd, usize, try_tcache, arena)); 1749} 1750 1751JEMALLOC_ALWAYS_INLINE_C void * 1752imallocx_maybe_flags(tsd_t *tsd, size_t size, int flags, size_t usize, 1753 size_t alignment, bool zero, bool try_tcache, arena_t *arena) 1754{ 1755 1756 if (likely(flags == 0)) 1757 return (imalloc(tsd, size)); 1758 return (imallocx_flags(tsd, usize, alignment, zero, try_tcache, arena)); 1759} 1760 1761static void * 1762imallocx_prof_sample(tsd_t *tsd, size_t size, int flags, size_t usize, 1763 size_t alignment, bool zero, bool try_tcache, arena_t *arena) 1764{ 1765 void *p; 1766 1767 if (usize <= SMALL_MAXCLASS) { 1768 assert(((alignment == 0) ? s2u(LARGE_MINCLASS) : 1769 sa2u(LARGE_MINCLASS, alignment)) == LARGE_MINCLASS); 1770 p = imalloct(tsd, LARGE_MINCLASS, try_tcache, arena); 1771 if (p == NULL) 1772 return (NULL); 1773 arena_prof_promoted(p, usize); 1774 } else { 1775 p = imallocx_maybe_flags(tsd, size, flags, usize, alignment, 1776 zero, try_tcache, arena); 1777 } 1778 1779 return (p); 1780} 1781 1782JEMALLOC_ALWAYS_INLINE_C void * 1783imallocx_prof(tsd_t *tsd, size_t size, int flags, size_t *usize) 1784{ 1785 void *p; 1786 size_t alignment; 1787 bool zero; 1788 bool try_tcache; 1789 arena_t *arena; 1790 prof_tctx_t *tctx; 1791 1792 if (unlikely(imallocx_flags_decode(tsd, size, flags, usize, &alignment, 1793 &zero, &try_tcache, &arena))) 1794 return (NULL); 1795 tctx = prof_alloc_prep(tsd, *usize, true); 1796 if (likely((uintptr_t)tctx == (uintptr_t)1U)) { 1797 p = imallocx_maybe_flags(tsd, size, flags, *usize, alignment, 1798 zero, try_tcache, arena); 1799 } else if ((uintptr_t)tctx > (uintptr_t)1U) { 1800 p = imallocx_prof_sample(tsd, size, flags, *usize, alignment, 1801 zero, try_tcache, arena); 1802 } else 1803 p = NULL; 1804 if (unlikely(p == NULL)) { 1805 prof_alloc_rollback(tsd, tctx, true); 1806 return (NULL); 1807 } 1808 prof_malloc(p, *usize, tctx); 1809 1810 return (p); 1811} 1812 1813JEMALLOC_ALWAYS_INLINE_C void * 1814imallocx_no_prof(tsd_t *tsd, size_t size, int flags, size_t *usize) 1815{ 1816 size_t alignment; 1817 bool zero; 1818 bool try_tcache; 1819 arena_t *arena; 1820 1821 if (likely(flags == 0)) { 1822 if (config_stats || (config_valgrind && unlikely(in_valgrind))) 1823 *usize = s2u(size); 1824 return (imalloc(tsd, size)); 1825 } 1826 1827 if (unlikely(imallocx_flags_decode_hard(tsd, size, flags, usize, 1828 &alignment, &zero, &try_tcache, &arena))) 1829 return (NULL); 1830 return (imallocx_flags(tsd, *usize, alignment, zero, try_tcache, 1831 arena)); 1832} 1833 1834void * 1835je_mallocx(size_t size, int flags) 1836{ 1837 tsd_t *tsd; 1838 void *p; 1839 size_t usize; 1840 1841 assert(size != 0); 1842 1843 if (unlikely(malloc_init())) 1844 goto label_oom; 1845 tsd = tsd_fetch(); 1846 1847 if (config_prof && opt_prof) 1848 p = imallocx_prof(tsd, size, flags, &usize); 1849 else 1850 p = imallocx_no_prof(tsd, size, flags, &usize); 1851 if (unlikely(p == NULL)) 1852 goto label_oom; 1853 1854 if (config_stats) { 1855 assert(usize == isalloc(p, config_prof)); 1856 *tsd_thread_allocatedp_get(tsd) += usize; 1857 } 1858 UTRACE(0, size, p); 1859 JEMALLOC_VALGRIND_MALLOC(true, p, usize, MALLOCX_ZERO_GET(flags)); 1860 return (p); 1861label_oom: 1862 if (config_xmalloc && unlikely(opt_xmalloc)) { 1863 malloc_write("<jemalloc>: Error in mallocx(): out of memory\n"); 1864 abort(); 1865 } 1866 UTRACE(0, size, 0); 1867 return (NULL); 1868} 1869 1870static void * 1871irallocx_prof_sample(tsd_t *tsd, void *oldptr, size_t size, size_t alignment, 1872 size_t usize, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, 1873 arena_t *arena, prof_tctx_t *tctx) 1874{ 1875 void *p; 1876 1877 if (tctx == NULL) 1878 return (NULL); 1879 if (usize <= SMALL_MAXCLASS) { 1880 p = iralloct(tsd, oldptr, LARGE_MINCLASS, alignment, zero, 1881 try_tcache_alloc, try_tcache_dalloc, arena); 1882 if (p == NULL) 1883 return (NULL); 1884 arena_prof_promoted(p, usize); 1885 } else { 1886 p = iralloct(tsd, oldptr, size, alignment, zero, 1887 try_tcache_alloc, try_tcache_dalloc, arena); 1888 } 1889 1890 return (p); 1891} 1892 1893JEMALLOC_ALWAYS_INLINE_C void * 1894irallocx_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t size, 1895 size_t alignment, size_t *usize, bool zero, bool try_tcache_alloc, 1896 bool try_tcache_dalloc, arena_t *arena) 1897{ 1898 void *p; 1899 prof_tctx_t *old_tctx, *tctx; 1900 1901 old_tctx = prof_tctx_get(oldptr); 1902 tctx = prof_alloc_prep(tsd, *usize, false); 1903 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { 1904 p = irallocx_prof_sample(tsd, oldptr, size, alignment, *usize, 1905 zero, try_tcache_alloc, try_tcache_dalloc, arena, tctx); 1906 } else { 1907 p = iralloct(tsd, oldptr, size, alignment, zero, 1908 try_tcache_alloc, try_tcache_dalloc, arena); 1909 } 1910 if (unlikely(p == NULL)) { 1911 prof_alloc_rollback(tsd, tctx, false); 1912 return (NULL); 1913 } 1914 1915 if (p == oldptr && alignment != 0) { 1916 /* 1917 * The allocation did not move, so it is possible that the size 1918 * class is smaller than would guarantee the requested 1919 * alignment, and that the alignment constraint was 1920 * serendipitously satisfied. Additionally, old_usize may not 1921 * be the same as the current usize because of in-place large 1922 * reallocation. Therefore, query the actual value of usize. 1923 */ 1924 *usize = isalloc(p, config_prof); 1925 } 1926 prof_realloc(tsd, p, *usize, tctx, false, old_usize, old_tctx); 1927 1928 return (p); 1929} 1930 1931void * 1932je_rallocx(void *ptr, size_t size, int flags) 1933{ 1934 void *p; 1935 tsd_t *tsd; 1936 size_t usize; 1937 UNUSED size_t old_usize JEMALLOC_CC_SILENCE_INIT(0); 1938 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 1939 size_t alignment = MALLOCX_ALIGN_GET(flags); 1940 bool zero = flags & MALLOCX_ZERO; 1941 bool try_tcache_alloc, try_tcache_dalloc; 1942 arena_t *arena; 1943 1944 assert(ptr != NULL); 1945 assert(size != 0); 1946 assert(malloc_initialized || IS_INITIALIZER); 1947 malloc_thread_init(); 1948 tsd = tsd_fetch(); 1949 1950 if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { 1951 unsigned arena_ind = MALLOCX_ARENA_GET(flags); 1952 arena_chunk_t *chunk; 1953 try_tcache_alloc = false; 1954 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1955 arena = arena_get(tsd, arena_ind, true, true); 1956 if (unlikely(arena == NULL)) 1957 goto label_oom; 1958 try_tcache_dalloc = (chunk == ptr || chunk->arena != arena); 1959 } else { 1960 try_tcache_alloc = true; 1961 try_tcache_dalloc = true; 1962 arena = NULL; 1963 } 1964 1965 if ((config_prof && opt_prof) || config_stats || 1966 ((config_valgrind && unlikely(in_valgrind)))) 1967 old_usize = isalloc(ptr, config_prof); 1968 if (config_valgrind && unlikely(in_valgrind)) 1969 old_rzsize = u2rz(old_usize); 1970 1971 if (config_prof && opt_prof) { 1972 usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); 1973 assert(usize != 0); 1974 p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize, 1975 zero, try_tcache_alloc, try_tcache_dalloc, arena); 1976 if (unlikely(p == NULL)) 1977 goto label_oom; 1978 } else { 1979 p = iralloct(tsd, ptr, size, alignment, zero, try_tcache_alloc, 1980 try_tcache_dalloc, arena); 1981 if (unlikely(p == NULL)) 1982 goto label_oom; 1983 if (config_stats || (config_valgrind && unlikely(in_valgrind))) 1984 usize = isalloc(p, config_prof); 1985 } 1986 1987 if (config_stats) { 1988 *tsd_thread_allocatedp_get(tsd) += usize; 1989 *tsd_thread_deallocatedp_get(tsd) += old_usize; 1990 } 1991 UTRACE(ptr, size, p); 1992 JEMALLOC_VALGRIND_REALLOC(true, p, usize, false, ptr, old_usize, 1993 old_rzsize, false, zero); 1994 return (p); 1995label_oom: 1996 if (config_xmalloc && unlikely(opt_xmalloc)) { 1997 malloc_write("<jemalloc>: Error in rallocx(): out of memory\n"); 1998 abort(); 1999 } 2000 UTRACE(ptr, size, 0); 2001 return (NULL); 2002} 2003 2004JEMALLOC_ALWAYS_INLINE_C size_t 2005ixallocx_helper(void *ptr, size_t old_usize, size_t size, size_t extra, 2006 size_t alignment, bool zero, arena_t *arena) 2007{ 2008 size_t usize; 2009 2010 if (ixalloc(ptr, size, extra, alignment, zero)) 2011 return (old_usize); 2012 usize = isalloc(ptr, config_prof); 2013 2014 return (usize); 2015} 2016 2017static size_t 2018ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra, 2019 size_t alignment, size_t max_usize, bool zero, arena_t *arena, 2020 prof_tctx_t *tctx) 2021{ 2022 size_t usize; 2023 2024 if (tctx == NULL) 2025 return (old_usize); 2026 /* Use minimum usize to determine whether promotion may happen. */ 2027 if (((alignment == 0) ? s2u(size) : sa2u(size, alignment)) <= 2028 SMALL_MAXCLASS) { 2029 if (ixalloc(ptr, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >= 2030 size+extra) ? 0 : size+extra - (SMALL_MAXCLASS+1), 2031 alignment, zero)) 2032 return (old_usize); 2033 usize = isalloc(ptr, config_prof); 2034 if (max_usize < LARGE_MINCLASS) 2035 arena_prof_promoted(ptr, usize); 2036 } else { 2037 usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, 2038 zero, arena); 2039 } 2040 2041 return (usize); 2042} 2043 2044JEMALLOC_ALWAYS_INLINE_C size_t 2045ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, 2046 size_t extra, size_t alignment, bool zero, arena_t *arena) 2047{ 2048 size_t max_usize, usize; 2049 prof_tctx_t *old_tctx, *tctx; 2050 2051 old_tctx = prof_tctx_get(ptr); 2052 /* 2053 * usize isn't knowable before ixalloc() returns when extra is non-zero. 2054 * Therefore, compute its maximum possible value and use that in 2055 * prof_alloc_prep() to decide whether to capture a backtrace. 2056 * prof_realloc() will use the actual usize to decide whether to sample. 2057 */ 2058 max_usize = (alignment == 0) ? s2u(size+extra) : sa2u(size+extra, 2059 alignment); 2060 tctx = prof_alloc_prep(tsd, max_usize, false); 2061 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { 2062 usize = ixallocx_prof_sample(ptr, old_usize, size, extra, 2063 alignment, zero, max_usize, arena, tctx); 2064 } else { 2065 usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, 2066 zero, arena); 2067 } 2068 if (unlikely(usize == old_usize)) { 2069 prof_alloc_rollback(tsd, tctx, false); 2070 return (usize); 2071 } 2072 prof_realloc(tsd, ptr, usize, tctx, false, old_usize, old_tctx); 2073 2074 return (usize); 2075} 2076 2077size_t 2078je_xallocx(void *ptr, size_t size, size_t extra, int flags) 2079{ 2080 tsd_t *tsd; 2081 size_t usize, old_usize; 2082 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 2083 size_t alignment = MALLOCX_ALIGN_GET(flags); 2084 bool zero = flags & MALLOCX_ZERO; 2085 arena_t *arena; 2086 2087 assert(ptr != NULL); 2088 assert(size != 0); 2089 assert(SIZE_T_MAX - size >= extra); 2090 assert(malloc_initialized || IS_INITIALIZER); 2091 malloc_thread_init(); 2092 tsd = tsd_fetch(); 2093 2094 if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { 2095 unsigned arena_ind = MALLOCX_ARENA_GET(flags); 2096 arena = arena_get(tsd, arena_ind, true, true); 2097 } else 2098 arena = NULL; 2099 2100 old_usize = isalloc(ptr, config_prof); 2101 if (config_valgrind && unlikely(in_valgrind)) 2102 old_rzsize = u2rz(old_usize); 2103 2104 if (config_prof && opt_prof) { 2105 usize = ixallocx_prof(tsd, ptr, old_usize, size, extra, 2106 alignment, zero, arena); 2107 } else { 2108 usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, 2109 zero, arena); 2110 } 2111 if (unlikely(usize == old_usize)) 2112 goto label_not_resized; 2113 2114 if (config_stats) { 2115 *tsd_thread_allocatedp_get(tsd) += usize; 2116 *tsd_thread_deallocatedp_get(tsd) += old_usize; 2117 } 2118 JEMALLOC_VALGRIND_REALLOC(false, ptr, usize, false, ptr, old_usize, 2119 old_rzsize, false, zero); 2120label_not_resized: 2121 UTRACE(ptr, size, ptr); 2122 return (usize); 2123} 2124 2125size_t 2126je_sallocx(const void *ptr, int flags) 2127{ 2128 size_t usize; 2129 2130 assert(malloc_initialized || IS_INITIALIZER); 2131 malloc_thread_init(); 2132 2133 if (config_ivsalloc) 2134 usize = ivsalloc(ptr, config_prof); 2135 else { 2136 assert(ptr != NULL); 2137 usize = isalloc(ptr, config_prof); 2138 } 2139 2140 return (usize); 2141} 2142 2143void 2144je_dallocx(void *ptr, int flags) 2145{ 2146 tsd_t *tsd; 2147 bool try_tcache; 2148 2149 assert(ptr != NULL); 2150 assert(malloc_initialized || IS_INITIALIZER); 2151 2152 tsd = tsd_fetch(); 2153 if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { 2154 unsigned arena_ind = MALLOCX_ARENA_GET(flags); 2155 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 2156 arena_t *arena = arena_get(tsd, arena_ind, true, true); 2157 /* 2158 * If arena is NULL, the application passed an arena that has 2159 * never been used before, which is unsupported during 2160 * deallocation. 2161 */ 2162 assert(arena != NULL); 2163 try_tcache = (chunk == ptr || chunk->arena != arena); 2164 } else 2165 try_tcache = true; 2166 2167 UTRACE(ptr, 0, 0); 2168 ifree(tsd_fetch(), ptr, try_tcache); 2169} 2170 2171JEMALLOC_ALWAYS_INLINE_C size_t 2172inallocx(size_t size, int flags) 2173{ 2174 size_t usize; 2175 2176 if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0)) 2177 usize = s2u(size); 2178 else 2179 usize = sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags)); 2180 assert(usize != 0); 2181 return (usize); 2182} 2183 2184void 2185je_sdallocx(void *ptr, size_t size, int flags) 2186{ 2187 tsd_t *tsd; 2188 bool try_tcache; 2189 size_t usize; 2190 2191 assert(ptr != NULL); 2192 assert(malloc_initialized || IS_INITIALIZER); 2193 usize = inallocx(size, flags); 2194 assert(usize == isalloc(ptr, config_prof)); 2195 2196 tsd = tsd_fetch(); 2197 if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { 2198 unsigned arena_ind = MALLOCX_ARENA_GET(flags); 2199 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 2200 arena_t *arena = arena_get(tsd, arena_ind, true, true); 2201 /* 2202 * If arena is NULL, the application passed an arena that has 2203 * never been used before, which is unsupported during 2204 * deallocation. 2205 */ 2206 try_tcache = (chunk == ptr || chunk->arena != arena); 2207 } else 2208 try_tcache = true; 2209 2210 UTRACE(ptr, 0, 0); 2211 isfree(tsd, ptr, usize, try_tcache); 2212} 2213 2214size_t 2215je_nallocx(size_t size, int flags) 2216{ 2217 2218 assert(size != 0); 2219 2220 if (unlikely(malloc_init())) 2221 return (0); 2222 2223 return (inallocx(size, flags)); 2224} 2225 2226int 2227je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, 2228 size_t newlen) 2229{ 2230 2231 if (unlikely(malloc_init())) 2232 return (EAGAIN); 2233 2234 return (ctl_byname(name, oldp, oldlenp, newp, newlen)); 2235} 2236 2237int 2238je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) 2239{ 2240 2241 if (unlikely(malloc_init())) 2242 return (EAGAIN); 2243 2244 return (ctl_nametomib(name, mibp, miblenp)); 2245} 2246 2247int 2248je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, 2249 void *newp, size_t newlen) 2250{ 2251 2252 if (unlikely(malloc_init())) 2253 return (EAGAIN); 2254 2255 return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen)); 2256} 2257 2258void 2259je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, 2260 const char *opts) 2261{ 2262 2263 stats_print(write_cb, cbopaque, opts); 2264} 2265 2266size_t 2267je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) 2268{ 2269 size_t ret; 2270 2271 assert(malloc_initialized || IS_INITIALIZER); 2272 malloc_thread_init(); 2273 2274 if (config_ivsalloc) 2275 ret = ivsalloc(ptr, config_prof); 2276 else 2277 ret = (ptr != NULL) ? isalloc(ptr, config_prof) : 0; 2278 2279 return (ret); 2280} 2281 2282/* 2283 * End non-standard functions. 2284 */ 2285/******************************************************************************/ 2286/* 2287 * The following functions are used by threading libraries for protection of 2288 * malloc during fork(). 2289 */ 2290 2291/* 2292 * If an application creates a thread before doing any allocation in the main 2293 * thread, then calls fork(2) in the main thread followed by memory allocation 2294 * in the child process, a race can occur that results in deadlock within the 2295 * child: the main thread may have forked while the created thread had 2296 * partially initialized the allocator. Ordinarily jemalloc prevents 2297 * fork/malloc races via the following functions it registers during 2298 * initialization using pthread_atfork(), but of course that does no good if 2299 * the allocator isn't fully initialized at fork time. The following library 2300 * constructor is a partial solution to this problem. It may still be possible 2301 * to trigger the deadlock described above, but doing so would involve forking 2302 * via a library constructor that runs before jemalloc's runs. 2303 */ 2304JEMALLOC_ATTR(constructor) 2305static void 2306jemalloc_constructor(void) 2307{ 2308 2309 malloc_init(); 2310} 2311 2312#ifndef JEMALLOC_MUTEX_INIT_CB 2313void 2314jemalloc_prefork(void) 2315#else 2316JEMALLOC_EXPORT void 2317_malloc_prefork(void) 2318#endif 2319{ 2320 unsigned i; 2321 2322#ifdef JEMALLOC_MUTEX_INIT_CB 2323 if (!malloc_initialized) 2324 return; 2325#endif 2326 assert(malloc_initialized); 2327 2328 /* Acquire all mutexes in a safe order. */ 2329 ctl_prefork(); 2330 prof_prefork(); 2331 malloc_mutex_prefork(&arenas_lock); 2332 for (i = 0; i < narenas_total; i++) { 2333 if (arenas[i] != NULL) 2334 arena_prefork(arenas[i]); 2335 } 2336 chunk_prefork(); 2337 base_prefork(); 2338 huge_prefork(); 2339} 2340 2341#ifndef JEMALLOC_MUTEX_INIT_CB 2342void 2343jemalloc_postfork_parent(void) 2344#else 2345JEMALLOC_EXPORT void 2346_malloc_postfork(void) 2347#endif 2348{ 2349 unsigned i; 2350 2351#ifdef JEMALLOC_MUTEX_INIT_CB 2352 if (!malloc_initialized) 2353 return; 2354#endif 2355 assert(malloc_initialized); 2356 2357 /* Release all mutexes, now that fork() has completed. */ 2358 huge_postfork_parent(); 2359 base_postfork_parent(); 2360 chunk_postfork_parent(); 2361 for (i = 0; i < narenas_total; i++) { 2362 if (arenas[i] != NULL) 2363 arena_postfork_parent(arenas[i]); 2364 } 2365 malloc_mutex_postfork_parent(&arenas_lock); 2366 prof_postfork_parent(); 2367 ctl_postfork_parent(); 2368} 2369 2370void 2371jemalloc_postfork_child(void) 2372{ 2373 unsigned i; 2374 2375 assert(malloc_initialized); 2376 2377 /* Release all mutexes, now that fork() has completed. */ 2378 huge_postfork_child(); 2379 base_postfork_child(); 2380 chunk_postfork_child(); 2381 for (i = 0; i < narenas_total; i++) { 2382 if (arenas[i] != NULL) 2383 arena_postfork_child(arenas[i]); 2384 } 2385 malloc_mutex_postfork_child(&arenas_lock); 2386 prof_postfork_child(); 2387 ctl_postfork_child(); 2388} 2389 2390/******************************************************************************/ 2391