1#define JEMALLOC_C_ 2#include "jemalloc/internal/jemalloc_internal.h" 3 4/******************************************************************************/ 5/* Data. */ 6 7malloc_tsd_data(, arenas, arena_t *, NULL) 8malloc_tsd_data(, thread_allocated, thread_allocated_t, 9 THREAD_ALLOCATED_INITIALIZER) 10 11/* Runtime configuration options. */ 12const char *je_malloc_conf; 13bool opt_abort = 14#ifdef JEMALLOC_DEBUG 15 true 16#else 17 false 18#endif 19 ; 20bool opt_junk = 21#if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) 22 true 23#else 24 false 25#endif 26 ; 27size_t opt_quarantine = ZU(0); 28bool opt_redzone = false; 29bool opt_utrace = false; 30bool opt_xmalloc = false; 31bool opt_zero = false; 32size_t opt_narenas = 0; 33 34/* Initialized to true if the process is running inside Valgrind. */ 35bool in_valgrind; 36 37unsigned ncpus; 38 39malloc_mutex_t arenas_lock; 40arena_t **arenas; 41unsigned narenas_total; 42unsigned narenas_auto; 43 44/* Set to true once the allocator has been initialized. */ 45static bool malloc_initialized = false; 46 47#ifdef JEMALLOC_THREADED_INIT 48/* Used to let the initializing thread recursively allocate. */ 49# define NO_INITIALIZER ((unsigned long)0) 50# define INITIALIZER pthread_self() 51# define IS_INITIALIZER (malloc_initializer == pthread_self()) 52static pthread_t malloc_initializer = NO_INITIALIZER; 53#else 54# define NO_INITIALIZER false 55# define INITIALIZER true 56# define IS_INITIALIZER malloc_initializer 57static bool malloc_initializer = NO_INITIALIZER; 58#endif 59 60/* Used to avoid initialization races. */ 61#ifdef _WIN32 62static malloc_mutex_t init_lock; 63 64JEMALLOC_ATTR(constructor) 65static void WINAPI 66_init_init_lock(void) 67{ 68 69 malloc_mutex_init(&init_lock); 70} 71 72#ifdef _MSC_VER 73# pragma section(".CRT$XCU", read) 74JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used) 75static const void (WINAPI *init_init_lock)(void) = _init_init_lock; 76#endif 77 78#else 79static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER; 80#endif 81 82typedef struct { 83 void *p; /* Input pointer (as in realloc(p, s)). */ 84 size_t s; /* Request size. */ 85 void *r; /* Result pointer. */ 86} malloc_utrace_t; 87 88#ifdef JEMALLOC_UTRACE 89# define UTRACE(a, b, c) do { \ 90 if (opt_utrace) { \ 91 int utrace_serrno = errno; \ 92 malloc_utrace_t ut; \ 93 ut.p = (a); \ 94 ut.s = (b); \ 95 ut.r = (c); \ 96 utrace(&ut, sizeof(ut)); \ 97 errno = utrace_serrno; \ 98 } \ 99} while (0) 100#else 101# define UTRACE(a, b, c) 102#endif 103 104/******************************************************************************/ 105/* 106 * Function prototypes for static functions that are referenced prior to 107 * definition. 108 */ 109 110static bool malloc_init_hard(void); 111 112/******************************************************************************/ 113/* 114 * Begin miscellaneous support functions. 115 */ 116 117/* Create a new arena and insert it into the arenas array at index ind. */ 118arena_t * 119arenas_extend(unsigned ind) 120{ 121 arena_t *ret; 122 123 ret = (arena_t *)base_alloc(sizeof(arena_t)); 124 if (ret != NULL && arena_new(ret, ind) == false) { 125 arenas[ind] = ret; 126 return (ret); 127 } 128 /* Only reached if there is an OOM error. */ 129 130 /* 131 * OOM here is quite inconvenient to propagate, since dealing with it 132 * would require a check for failure in the fast path. Instead, punt 133 * by using arenas[0]. In practice, this is an extremely unlikely 134 * failure. 135 */ 136 malloc_write("<jemalloc>: Error initializing arena\n"); 137 if (opt_abort) 138 abort(); 139 140 return (arenas[0]); 141} 142 143/* Slow path, called only by choose_arena(). */ 144arena_t * 145choose_arena_hard(void) 146{ 147 arena_t *ret; 148 149 if (narenas_auto > 1) { 150 unsigned i, choose, first_null; 151 152 choose = 0; 153 first_null = narenas_auto; 154 malloc_mutex_lock(&arenas_lock); 155 assert(arenas[0] != NULL); 156 for (i = 1; i < narenas_auto; i++) { 157 if (arenas[i] != NULL) { 158 /* 159 * Choose the first arena that has the lowest 160 * number of threads assigned to it. 161 */ 162 if (arenas[i]->nthreads < 163 arenas[choose]->nthreads) 164 choose = i; 165 } else if (first_null == narenas_auto) { 166 /* 167 * Record the index of the first uninitialized 168 * arena, in case all extant arenas are in use. 169 * 170 * NB: It is possible for there to be 171 * discontinuities in terms of initialized 172 * versus uninitialized arenas, due to the 173 * "thread.arena" mallctl. 174 */ 175 first_null = i; 176 } 177 } 178 179 if (arenas[choose]->nthreads == 0 180 || first_null == narenas_auto) { 181 /* 182 * Use an unloaded arena, or the least loaded arena if 183 * all arenas are already initialized. 184 */ 185 ret = arenas[choose]; 186 } else { 187 /* Initialize a new arena. */ 188 ret = arenas_extend(first_null); 189 } 190 ret->nthreads++; 191 malloc_mutex_unlock(&arenas_lock); 192 } else { 193 ret = arenas[0]; 194 malloc_mutex_lock(&arenas_lock); 195 ret->nthreads++; 196 malloc_mutex_unlock(&arenas_lock); 197 } 198 199 arenas_tsd_set(&ret); 200 201 return (ret); 202} 203 204static void 205stats_print_atexit(void) 206{ 207 208 if (config_tcache && config_stats) { 209 unsigned narenas, i; 210 211 /* 212 * Merge stats from extant threads. This is racy, since 213 * individual threads do not lock when recording tcache stats 214 * events. As a consequence, the final stats may be slightly 215 * out of date by the time they are reported, if other threads 216 * continue to allocate. 217 */ 218 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { 219 arena_t *arena = arenas[i]; 220 if (arena != NULL) { 221 tcache_t *tcache; 222 223 /* 224 * tcache_stats_merge() locks bins, so if any 225 * code is introduced that acquires both arena 226 * and bin locks in the opposite order, 227 * deadlocks may result. 228 */ 229 malloc_mutex_lock(&arena->lock); 230 ql_foreach(tcache, &arena->tcache_ql, link) { 231 tcache_stats_merge(tcache, arena); 232 } 233 malloc_mutex_unlock(&arena->lock); 234 } 235 } 236 } 237 je_malloc_stats_print(NULL, NULL, NULL); 238} 239 240/* 241 * End miscellaneous support functions. 242 */ 243/******************************************************************************/ 244/* 245 * Begin initialization functions. 246 */ 247 248static unsigned 249malloc_ncpus(void) 250{ 251 long result; 252 253#ifdef _WIN32 254 SYSTEM_INFO si; 255 GetSystemInfo(&si); 256 result = si.dwNumberOfProcessors; 257#else 258 result = sysconf(_SC_NPROCESSORS_ONLN); 259#endif 260 return ((result == -1) ? 1 : (unsigned)result); 261} 262 263void 264arenas_cleanup(void *arg) 265{ 266 arena_t *arena = *(arena_t **)arg; 267 268 malloc_mutex_lock(&arenas_lock); 269 arena->nthreads--; 270 malloc_mutex_unlock(&arenas_lock); 271} 272 273JEMALLOC_ALWAYS_INLINE_C void 274malloc_thread_init(void) 275{ 276 277 /* 278 * TSD initialization can't be safely done as a side effect of 279 * deallocation, because it is possible for a thread to do nothing but 280 * deallocate its TLS data via free(), in which case writing to TLS 281 * would cause write-after-free memory corruption. The quarantine 282 * facility *only* gets used as a side effect of deallocation, so make 283 * a best effort attempt at initializing its TSD by hooking all 284 * allocation events. 285 */ 286 if (config_fill && opt_quarantine) 287 quarantine_alloc_hook(); 288} 289 290JEMALLOC_ALWAYS_INLINE_C bool 291malloc_init(void) 292{ 293 294 if (malloc_initialized == false && malloc_init_hard()) 295 return (true); 296 malloc_thread_init(); 297 298 return (false); 299} 300 301static bool 302malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, 303 char const **v_p, size_t *vlen_p) 304{ 305 bool accept; 306 const char *opts = *opts_p; 307 308 *k_p = opts; 309 310 for (accept = false; accept == false;) { 311 switch (*opts) { 312 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': 313 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': 314 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': 315 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': 316 case 'Y': case 'Z': 317 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': 318 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': 319 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': 320 case 's': case 't': case 'u': case 'v': case 'w': case 'x': 321 case 'y': case 'z': 322 case '0': case '1': case '2': case '3': case '4': case '5': 323 case '6': case '7': case '8': case '9': 324 case '_': 325 opts++; 326 break; 327 case ':': 328 opts++; 329 *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p; 330 *v_p = opts; 331 accept = true; 332 break; 333 case '\0': 334 if (opts != *opts_p) { 335 malloc_write("<jemalloc>: Conf string ends " 336 "with key\n"); 337 } 338 return (true); 339 default: 340 malloc_write("<jemalloc>: Malformed conf string\n"); 341 return (true); 342 } 343 } 344 345 for (accept = false; accept == false;) { 346 switch (*opts) { 347 case ',': 348 opts++; 349 /* 350 * Look ahead one character here, because the next time 351 * this function is called, it will assume that end of 352 * input has been cleanly reached if no input remains, 353 * but we have optimistically already consumed the 354 * comma if one exists. 355 */ 356 if (*opts == '\0') { 357 malloc_write("<jemalloc>: Conf string ends " 358 "with comma\n"); 359 } 360 *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p; 361 accept = true; 362 break; 363 case '\0': 364 *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p; 365 accept = true; 366 break; 367 default: 368 opts++; 369 break; 370 } 371 } 372 373 *opts_p = opts; 374 return (false); 375} 376 377static void 378malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, 379 size_t vlen) 380{ 381 382 malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k, 383 (int)vlen, v); 384} 385 386static void 387malloc_conf_init(void) 388{ 389 unsigned i; 390 char buf[PATH_MAX + 1]; 391 const char *opts, *k, *v; 392 size_t klen, vlen; 393 394 /* 395 * Automatically configure valgrind before processing options. The 396 * valgrind option remains in jemalloc 3.x for compatibility reasons. 397 */ 398 if (config_valgrind) { 399 in_valgrind = (RUNNING_ON_VALGRIND != 0) ? true : false; 400 if (config_fill && in_valgrind) { 401 opt_junk = false; 402 assert(opt_zero == false); 403 opt_quarantine = JEMALLOC_VALGRIND_QUARANTINE_DEFAULT; 404 opt_redzone = true; 405 } 406 if (config_tcache && in_valgrind) 407 opt_tcache = false; 408 } 409 410#if defined(__ANDROID__) 411 /* Android only supports compiled options. */ 412 for (i = 0; i < 1; i++) { 413#else 414 for (i = 0; i < 3; i++) { 415#endif 416 /* Get runtime configuration. */ 417 switch (i) { 418 case 0: 419 if (je_malloc_conf != NULL) { 420 /* 421 * Use options that were compiled into the 422 * program. 423 */ 424 opts = je_malloc_conf; 425 } else { 426 /* No configuration specified. */ 427 buf[0] = '\0'; 428 opts = buf; 429 } 430 break; 431 case 1: { 432 int linklen = 0; 433#ifndef _WIN32 434 int saved_errno = errno; 435 const char *linkname = 436# ifdef JEMALLOC_PREFIX 437 "/etc/"JEMALLOC_PREFIX"malloc.conf" 438# else 439 "/etc/malloc.conf" 440# endif 441 ; 442 443 /* 444 * Try to use the contents of the "/etc/malloc.conf" 445 * symbolic link's name. 446 */ 447 linklen = readlink(linkname, buf, sizeof(buf) - 1); 448 if (linklen == -1) { 449 /* No configuration specified. */ 450 linklen = 0; 451 /* restore errno */ 452 set_errno(saved_errno); 453 } 454#endif 455 buf[linklen] = '\0'; 456 opts = buf; 457 break; 458 } case 2: { 459 const char *envname = 460#ifdef JEMALLOC_PREFIX 461 JEMALLOC_CPREFIX"MALLOC_CONF" 462#else 463 "MALLOC_CONF" 464#endif 465 ; 466 467 if ((opts = getenv(envname)) != NULL) { 468 /* 469 * Do nothing; opts is already initialized to 470 * the value of the MALLOC_CONF environment 471 * variable. 472 */ 473 } else { 474 /* No configuration specified. */ 475 buf[0] = '\0'; 476 opts = buf; 477 } 478 break; 479 } default: 480 not_reached(); 481 buf[0] = '\0'; 482 opts = buf; 483 } 484 485 while (*opts != '\0' && malloc_conf_next(&opts, &k, &klen, &v, 486 &vlen) == false) { 487#define CONF_MATCH(n) \ 488 (sizeof(n)-1 == klen && strncmp(n, k, klen) == 0) 489#define CONF_HANDLE_BOOL(o, n, cont) \ 490 if (CONF_MATCH(n)) { \ 491 if (strncmp("true", v, vlen) == 0 && \ 492 vlen == sizeof("true")-1) \ 493 o = true; \ 494 else if (strncmp("false", v, vlen) == \ 495 0 && vlen == sizeof("false")-1) \ 496 o = false; \ 497 else { \ 498 malloc_conf_error( \ 499 "Invalid conf value", \ 500 k, klen, v, vlen); \ 501 } \ 502 if (cont) \ 503 continue; \ 504 } 505#define CONF_HANDLE_SIZE_T(o, n, min, max, clip) \ 506 if (CONF_MATCH(n)) { \ 507 uintmax_t um; \ 508 char *end; \ 509 \ 510 set_errno(0); \ 511 um = malloc_strtoumax(v, &end, 0); \ 512 if (get_errno() != 0 || (uintptr_t)end -\ 513 (uintptr_t)v != vlen) { \ 514 malloc_conf_error( \ 515 "Invalid conf value", \ 516 k, klen, v, vlen); \ 517 } else if (clip) { \ 518 if (min != 0 && um < min) \ 519 o = min; \ 520 else if (um > max) \ 521 o = max; \ 522 else \ 523 o = um; \ 524 } else { \ 525 if ((min != 0 && um < min) || \ 526 um > max) { \ 527 malloc_conf_error( \ 528 "Out-of-range " \ 529 "conf value", \ 530 k, klen, v, vlen); \ 531 } else \ 532 o = um; \ 533 } \ 534 continue; \ 535 } 536#define CONF_HANDLE_SSIZE_T(o, n, min, max) \ 537 if (CONF_MATCH(n)) { \ 538 long l; \ 539 char *end; \ 540 \ 541 set_errno(0); \ 542 l = strtol(v, &end, 0); \ 543 if (get_errno() != 0 || (uintptr_t)end -\ 544 (uintptr_t)v != vlen) { \ 545 malloc_conf_error( \ 546 "Invalid conf value", \ 547 k, klen, v, vlen); \ 548 } else if (l < (ssize_t)min || l > \ 549 (ssize_t)max) { \ 550 malloc_conf_error( \ 551 "Out-of-range conf value", \ 552 k, klen, v, vlen); \ 553 } else \ 554 o = l; \ 555 continue; \ 556 } 557#define CONF_HANDLE_CHAR_P(o, n, d) \ 558 if (CONF_MATCH(n)) { \ 559 size_t cpylen = (vlen <= \ 560 sizeof(o)-1) ? vlen : \ 561 sizeof(o)-1; \ 562 strncpy(o, v, cpylen); \ 563 o[cpylen] = '\0'; \ 564 continue; \ 565 } 566 567 CONF_HANDLE_BOOL(opt_abort, "abort", true) 568 /* 569 * Chunks always require at least one header page, plus 570 * one data page in the absence of redzones, or three 571 * pages in the presence of redzones. In order to 572 * simplify options processing, fix the limit based on 573 * config_fill. 574 */ 575 CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE + 576 (config_fill ? 2 : 1), (sizeof(size_t) << 3) - 1, 577 true) 578 if (strncmp("dss", k, klen) == 0) { 579 int i; 580 bool match = false; 581 for (i = 0; i < dss_prec_limit; i++) { 582 if (strncmp(dss_prec_names[i], v, vlen) 583 == 0) { 584 if (chunk_dss_prec_set(i)) { 585 malloc_conf_error( 586 "Error setting dss", 587 k, klen, v, vlen); 588 } else { 589 opt_dss = 590 dss_prec_names[i]; 591 match = true; 592 break; 593 } 594 } 595 } 596 if (match == false) { 597 malloc_conf_error("Invalid conf value", 598 k, klen, v, vlen); 599 } 600 continue; 601 } 602 CONF_HANDLE_SIZE_T(opt_narenas, "narenas", 1, 603 SIZE_T_MAX, false) 604 CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult", 605 -1, (sizeof(size_t) << 3) - 1) 606 CONF_HANDLE_BOOL(opt_stats_print, "stats_print", true) 607 if (config_fill) { 608 CONF_HANDLE_BOOL(opt_junk, "junk", true) 609 CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine", 610 0, SIZE_T_MAX, false) 611 CONF_HANDLE_BOOL(opt_redzone, "redzone", true) 612 CONF_HANDLE_BOOL(opt_zero, "zero", true) 613 } 614 if (config_utrace) { 615 CONF_HANDLE_BOOL(opt_utrace, "utrace", true) 616 } 617 if (config_xmalloc) { 618 CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc", true) 619 } 620 if (config_tcache) { 621 CONF_HANDLE_BOOL(opt_tcache, "tcache", 622 !config_valgrind || !in_valgrind) 623 if (CONF_MATCH("tcache")) { 624 assert(config_valgrind && in_valgrind); 625 if (opt_tcache) { 626 opt_tcache = false; 627 malloc_conf_error( 628 "tcache cannot be enabled " 629 "while running inside Valgrind", 630 k, klen, v, vlen); 631 } 632 continue; 633 } 634 CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, 635 "lg_tcache_max", -1, 636 (sizeof(size_t) << 3) - 1) 637 } 638 if (config_prof) { 639 CONF_HANDLE_BOOL(opt_prof, "prof", true) 640 CONF_HANDLE_CHAR_P(opt_prof_prefix, 641 "prof_prefix", "jeprof") 642 CONF_HANDLE_BOOL(opt_prof_active, "prof_active", 643 true) 644 CONF_HANDLE_SSIZE_T(opt_lg_prof_sample, 645 "lg_prof_sample", 0, 646 (sizeof(uint64_t) << 3) - 1) 647 CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum", 648 true) 649 CONF_HANDLE_SSIZE_T(opt_lg_prof_interval, 650 "lg_prof_interval", -1, 651 (sizeof(uint64_t) << 3) - 1) 652 CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump", 653 true) 654 CONF_HANDLE_BOOL(opt_prof_final, "prof_final", 655 true) 656 CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak", 657 true) 658 } 659 malloc_conf_error("Invalid conf pair", k, klen, v, 660 vlen); 661#undef CONF_MATCH 662#undef CONF_HANDLE_BOOL 663#undef CONF_HANDLE_SIZE_T 664#undef CONF_HANDLE_SSIZE_T 665#undef CONF_HANDLE_CHAR_P 666 } 667 } 668} 669 670static bool 671malloc_init_hard(void) 672{ 673 arena_t *init_arenas[1]; 674 675 malloc_mutex_lock(&init_lock); 676 if (malloc_initialized || IS_INITIALIZER) { 677 /* 678 * Another thread initialized the allocator before this one 679 * acquired init_lock, or this thread is the initializing 680 * thread, and it is recursively allocating. 681 */ 682 malloc_mutex_unlock(&init_lock); 683 return (false); 684 } 685#ifdef JEMALLOC_THREADED_INIT 686 if (malloc_initializer != NO_INITIALIZER && IS_INITIALIZER == false) { 687 /* Busy-wait until the initializing thread completes. */ 688 do { 689 malloc_mutex_unlock(&init_lock); 690 CPU_SPINWAIT; 691 malloc_mutex_lock(&init_lock); 692 } while (malloc_initialized == false); 693 malloc_mutex_unlock(&init_lock); 694 return (false); 695 } 696#endif 697 malloc_initializer = INITIALIZER; 698 699 malloc_tsd_boot(); 700 if (config_prof) 701 prof_boot0(); 702 703 malloc_conf_init(); 704 705 if (opt_stats_print) { 706 /* Print statistics at exit. */ 707 if (atexit(stats_print_atexit) != 0) { 708 malloc_write("<jemalloc>: Error in atexit()\n"); 709 if (opt_abort) 710 abort(); 711 } 712 } 713 714 if (base_boot()) { 715 malloc_mutex_unlock(&init_lock); 716 return (true); 717 } 718 719 if (chunk_boot()) { 720 malloc_mutex_unlock(&init_lock); 721 return (true); 722 } 723 724 if (ctl_boot()) { 725 malloc_mutex_unlock(&init_lock); 726 return (true); 727 } 728 729 if (config_prof) 730 prof_boot1(); 731 732 arena_boot(); 733 734 if (config_tcache && tcache_boot0()) { 735 malloc_mutex_unlock(&init_lock); 736 return (true); 737 } 738 739 if (huge_boot()) { 740 malloc_mutex_unlock(&init_lock); 741 return (true); 742 } 743 744 if (malloc_mutex_init(&arenas_lock)) { 745 malloc_mutex_unlock(&init_lock); 746 return (true); 747 } 748 749 /* 750 * Create enough scaffolding to allow recursive allocation in 751 * malloc_ncpus(). 752 */ 753 narenas_total = narenas_auto = 1; 754 arenas = init_arenas; 755 memset(arenas, 0, sizeof(arena_t *) * narenas_auto); 756 757 /* 758 * Initialize one arena here. The rest are lazily created in 759 * choose_arena_hard(). 760 */ 761 arenas_extend(0); 762 if (arenas[0] == NULL) { 763 malloc_mutex_unlock(&init_lock); 764 return (true); 765 } 766 767 /* Initialize allocation counters before any allocations can occur. */ 768 if (config_stats && thread_allocated_tsd_boot()) { 769 malloc_mutex_unlock(&init_lock); 770 return (true); 771 } 772 773 if (arenas_tsd_boot()) { 774 malloc_mutex_unlock(&init_lock); 775 return (true); 776 } 777 778 if (config_tcache && tcache_boot1()) { 779 malloc_mutex_unlock(&init_lock); 780 return (true); 781 } 782 783 if (config_fill && quarantine_boot()) { 784 malloc_mutex_unlock(&init_lock); 785 return (true); 786 } 787 788 if (config_prof && prof_boot2()) { 789 malloc_mutex_unlock(&init_lock); 790 return (true); 791 } 792 793 malloc_mutex_unlock(&init_lock); 794 /**********************************************************************/ 795 /* Recursive allocation may follow. */ 796 797 ncpus = malloc_ncpus(); 798 799#if (!defined(JEMALLOC_MUTEX_INIT_CB) && !defined(JEMALLOC_ZONE) \ 800 && !defined(_WIN32) && !defined(__native_client__)) 801 /* LinuxThreads's pthread_atfork() allocates. */ 802 if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent, 803 jemalloc_postfork_child) != 0) { 804 malloc_write("<jemalloc>: Error in pthread_atfork()\n"); 805 if (opt_abort) 806 abort(); 807 } 808#endif 809 810 /* Done recursively allocating. */ 811 /**********************************************************************/ 812 malloc_mutex_lock(&init_lock); 813 814 if (mutex_boot()) { 815 malloc_mutex_unlock(&init_lock); 816 return (true); 817 } 818 819 if (opt_narenas == 0) { 820 /* 821 * For SMP systems, create more than one arena per CPU by 822 * default. 823 */ 824 if (ncpus > 1) 825 opt_narenas = ncpus << 2; 826 else 827 opt_narenas = 1; 828 } 829#if defined(ANDROID_MAX_ARENAS) 830 /* Never create more than MAX_ARENAS arenas regardless of num_cpus. 831 * Extra arenas use more PSS and are not very useful unless 832 * lots of threads are allocing/freeing at the same time. 833 */ 834 if (opt_narenas > ANDROID_MAX_ARENAS) 835 opt_narenas = ANDROID_MAX_ARENAS; 836#endif 837 narenas_auto = opt_narenas; 838 /* 839 * Make sure that the arenas array can be allocated. In practice, this 840 * limit is enough to allow the allocator to function, but the ctl 841 * machinery will fail to allocate memory at far lower limits. 842 */ 843 if (narenas_auto > chunksize / sizeof(arena_t *)) { 844 narenas_auto = chunksize / sizeof(arena_t *); 845 malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n", 846 narenas_auto); 847 } 848 narenas_total = narenas_auto; 849 850 /* Allocate and initialize arenas. */ 851 arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas_total); 852 if (arenas == NULL) { 853 malloc_mutex_unlock(&init_lock); 854 return (true); 855 } 856 /* 857 * Zero the array. In practice, this should always be pre-zeroed, 858 * since it was just mmap()ed, but let's be sure. 859 */ 860 memset(arenas, 0, sizeof(arena_t *) * narenas_total); 861 /* Copy the pointer to the one arena that was already initialized. */ 862 arenas[0] = init_arenas[0]; 863 864 malloc_initialized = true; 865 malloc_mutex_unlock(&init_lock); 866 867 return (false); 868} 869 870/* 871 * End initialization functions. 872 */ 873/******************************************************************************/ 874/* 875 * Begin malloc(3)-compatible functions. 876 */ 877 878static void * 879imalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt) 880{ 881 void *p; 882 883 if (cnt == NULL) 884 return (NULL); 885 if (usize <= SMALL_MAXCLASS) { 886 p = imalloc(SMALL_MAXCLASS+1); 887 if (p == NULL) 888 return (NULL); 889 arena_prof_promoted(p, usize); 890 } else 891 p = imalloc(usize); 892 893 return (p); 894} 895 896JEMALLOC_ALWAYS_INLINE_C void * 897imalloc_prof(size_t usize) 898{ 899 void *p; 900 prof_thr_cnt_t *cnt; 901 902 PROF_ALLOC_PREP(usize, cnt); 903 if ((uintptr_t)cnt != (uintptr_t)1U) 904 p = imalloc_prof_sample(usize, cnt); 905 else 906 p = imalloc(usize); 907 if (p == NULL) 908 return (NULL); 909 prof_malloc(p, usize, cnt); 910 911 return (p); 912} 913 914JEMALLOC_ALWAYS_INLINE_C void * 915imalloc_body(size_t size, size_t *usize) 916{ 917 918 if (malloc_init()) 919 return (NULL); 920 921 if (config_prof && opt_prof) { 922 *usize = s2u(size); 923 return (imalloc_prof(*usize)); 924 } 925 926 if (config_stats || (config_valgrind && in_valgrind)) 927 *usize = s2u(size); 928 return (imalloc(size)); 929} 930 931void * 932je_malloc(size_t size) 933{ 934 void *ret; 935 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 936 937 if (size == 0) 938 size = 1; 939 940 ret = imalloc_body(size, &usize); 941 if (ret == NULL) { 942 if (config_xmalloc && opt_xmalloc) { 943 malloc_write("<jemalloc>: Error in malloc(): " 944 "out of memory\n"); 945 abort(); 946 } 947 set_errno(ENOMEM); 948 } 949 if (config_stats && ret != NULL) { 950 assert(usize == isalloc(ret, config_prof)); 951 thread_allocated_tsd_get()->allocated += usize; 952 } 953 UTRACE(0, size, ret); 954 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, false); 955 return (ret); 956} 957 958static void * 959imemalign_prof_sample(size_t alignment, size_t usize, prof_thr_cnt_t *cnt) 960{ 961 void *p; 962 963 if (cnt == NULL) 964 return (NULL); 965 if (usize <= SMALL_MAXCLASS) { 966 assert(sa2u(SMALL_MAXCLASS+1, alignment) != 0); 967 p = ipalloc(sa2u(SMALL_MAXCLASS+1, alignment), alignment, 968 false); 969 if (p == NULL) 970 return (NULL); 971 arena_prof_promoted(p, usize); 972 } else 973 p = ipalloc(usize, alignment, false); 974 975 return (p); 976} 977 978JEMALLOC_ALWAYS_INLINE_C void * 979imemalign_prof(size_t alignment, size_t usize, prof_thr_cnt_t *cnt) 980{ 981 void *p; 982 983 if ((uintptr_t)cnt != (uintptr_t)1U) 984 p = imemalign_prof_sample(alignment, usize, cnt); 985 else 986 p = ipalloc(usize, alignment, false); 987 if (p == NULL) 988 return (NULL); 989 prof_malloc(p, usize, cnt); 990 991 return (p); 992} 993 994JEMALLOC_ATTR(nonnull(1)) 995static int 996imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment) 997{ 998 int ret; 999 size_t usize; 1000 void *result; 1001 1002 assert(min_alignment != 0); 1003 1004 if (malloc_init()) { 1005 result = NULL; 1006 goto label_oom; 1007 } else { 1008 if (size == 0) 1009 size = 1; 1010 1011 /* Make sure that alignment is a large enough power of 2. */ 1012 if (((alignment - 1) & alignment) != 0 1013 || (alignment < min_alignment)) { 1014 if (config_xmalloc && opt_xmalloc) { 1015 malloc_write("<jemalloc>: Error allocating " 1016 "aligned memory: invalid alignment\n"); 1017 abort(); 1018 } 1019 result = NULL; 1020 ret = EINVAL; 1021 goto label_return; 1022 } 1023 1024 usize = sa2u(size, alignment); 1025 if (usize == 0) { 1026 result = NULL; 1027 goto label_oom; 1028 } 1029 1030 if (config_prof && opt_prof) { 1031 prof_thr_cnt_t *cnt; 1032 1033 PROF_ALLOC_PREP(usize, cnt); 1034 result = imemalign_prof(alignment, usize, cnt); 1035 } else 1036 result = ipalloc(usize, alignment, false); 1037 if (result == NULL) 1038 goto label_oom; 1039 } 1040 1041 *memptr = result; 1042 ret = 0; 1043label_return: 1044 if (config_stats && result != NULL) { 1045 assert(usize == isalloc(result, config_prof)); 1046 thread_allocated_tsd_get()->allocated += usize; 1047 } 1048 UTRACE(0, size, result); 1049 return (ret); 1050label_oom: 1051 assert(result == NULL); 1052 if (config_xmalloc && opt_xmalloc) { 1053 malloc_write("<jemalloc>: Error allocating aligned memory: " 1054 "out of memory\n"); 1055 abort(); 1056 } 1057 ret = ENOMEM; 1058 goto label_return; 1059} 1060 1061int 1062je_posix_memalign(void **memptr, size_t alignment, size_t size) 1063{ 1064 int ret = imemalign(memptr, alignment, size, sizeof(void *)); 1065 JEMALLOC_VALGRIND_MALLOC(ret == 0, *memptr, isalloc(*memptr, 1066 config_prof), false); 1067 return (ret); 1068} 1069 1070void * 1071je_aligned_alloc(size_t alignment, size_t size) 1072{ 1073 void *ret; 1074 int err; 1075 1076 if ((err = imemalign(&ret, alignment, size, 1)) != 0) { 1077 ret = NULL; 1078 set_errno(err); 1079 } 1080 JEMALLOC_VALGRIND_MALLOC(err == 0, ret, isalloc(ret, config_prof), 1081 false); 1082 return (ret); 1083} 1084 1085static void * 1086icalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt) 1087{ 1088 void *p; 1089 1090 if (cnt == NULL) 1091 return (NULL); 1092 if (usize <= SMALL_MAXCLASS) { 1093 p = icalloc(SMALL_MAXCLASS+1); 1094 if (p == NULL) 1095 return (NULL); 1096 arena_prof_promoted(p, usize); 1097 } else 1098 p = icalloc(usize); 1099 1100 return (p); 1101} 1102 1103JEMALLOC_ALWAYS_INLINE_C void * 1104icalloc_prof(size_t usize, prof_thr_cnt_t *cnt) 1105{ 1106 void *p; 1107 1108 if ((uintptr_t)cnt != (uintptr_t)1U) 1109 p = icalloc_prof_sample(usize, cnt); 1110 else 1111 p = icalloc(usize); 1112 if (p == NULL) 1113 return (NULL); 1114 prof_malloc(p, usize, cnt); 1115 1116 return (p); 1117} 1118 1119void * 1120je_calloc(size_t num, size_t size) 1121{ 1122 void *ret; 1123 size_t num_size; 1124 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1125 1126 if (malloc_init()) { 1127 num_size = 0; 1128 ret = NULL; 1129 goto label_return; 1130 } 1131 1132 num_size = num * size; 1133 if (num_size == 0) { 1134 if (num == 0 || size == 0) 1135 num_size = 1; 1136 else { 1137 ret = NULL; 1138 goto label_return; 1139 } 1140 /* 1141 * Try to avoid division here. We know that it isn't possible to 1142 * overflow during multiplication if neither operand uses any of the 1143 * most significant half of the bits in a size_t. 1144 */ 1145 } else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2))) 1146 && (num_size / size != num)) { 1147 /* size_t overflow. */ 1148 ret = NULL; 1149 goto label_return; 1150 } 1151 1152 if (config_prof && opt_prof) { 1153 prof_thr_cnt_t *cnt; 1154 1155 usize = s2u(num_size); 1156 PROF_ALLOC_PREP(usize, cnt); 1157 ret = icalloc_prof(usize, cnt); 1158 } else { 1159 if (config_stats || (config_valgrind && in_valgrind)) 1160 usize = s2u(num_size); 1161 ret = icalloc(num_size); 1162 } 1163 1164label_return: 1165 if (ret == NULL) { 1166 if (config_xmalloc && opt_xmalloc) { 1167 malloc_write("<jemalloc>: Error in calloc(): out of " 1168 "memory\n"); 1169 abort(); 1170 } 1171 set_errno(ENOMEM); 1172 } 1173 if (config_stats && ret != NULL) { 1174 assert(usize == isalloc(ret, config_prof)); 1175 thread_allocated_tsd_get()->allocated += usize; 1176 } 1177 UTRACE(0, num_size, ret); 1178 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, true); 1179 return (ret); 1180} 1181 1182static void * 1183irealloc_prof_sample(void *oldptr, size_t usize, prof_thr_cnt_t *cnt) 1184{ 1185 void *p; 1186 1187 if (cnt == NULL) 1188 return (NULL); 1189 if (usize <= SMALL_MAXCLASS) { 1190 p = iralloc(oldptr, SMALL_MAXCLASS+1, 0, 0, false); 1191 if (p == NULL) 1192 return (NULL); 1193 arena_prof_promoted(p, usize); 1194 } else 1195 p = iralloc(oldptr, usize, 0, 0, false); 1196 1197 return (p); 1198} 1199 1200JEMALLOC_ALWAYS_INLINE_C void * 1201irealloc_prof(void *oldptr, size_t old_usize, size_t usize, prof_thr_cnt_t *cnt) 1202{ 1203 void *p; 1204 prof_ctx_t *old_ctx; 1205 1206 old_ctx = prof_ctx_get(oldptr); 1207 if ((uintptr_t)cnt != (uintptr_t)1U) 1208 p = irealloc_prof_sample(oldptr, usize, cnt); 1209 else 1210 p = iralloc(oldptr, usize, 0, 0, false); 1211 if (p == NULL) 1212 return (NULL); 1213 prof_realloc(p, usize, cnt, old_usize, old_ctx); 1214 1215 return (p); 1216} 1217 1218JEMALLOC_INLINE_C void 1219ifree(void *ptr) 1220{ 1221 size_t usize; 1222 UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); 1223 1224 assert(ptr != NULL); 1225 assert(malloc_initialized || IS_INITIALIZER); 1226 1227 if (config_prof && opt_prof) { 1228 usize = isalloc(ptr, config_prof); 1229 prof_free(ptr, usize); 1230 } else if (config_stats || config_valgrind) 1231 usize = isalloc(ptr, config_prof); 1232 if (config_stats) 1233 thread_allocated_tsd_get()->deallocated += usize; 1234 if (config_valgrind && in_valgrind) 1235 rzsize = p2rz(ptr); 1236 iqalloc(ptr); 1237 JEMALLOC_VALGRIND_FREE(ptr, rzsize); 1238} 1239 1240void * 1241je_realloc(void *ptr, size_t size) 1242{ 1243 void *ret; 1244 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1245 size_t old_usize = 0; 1246 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 1247 1248 if (size == 0) { 1249 if (ptr != NULL) { 1250 /* realloc(ptr, 0) is equivalent to free(ptr). */ 1251 UTRACE(ptr, 0, 0); 1252 ifree(ptr); 1253 return (NULL); 1254 } 1255 size = 1; 1256 } 1257 1258 if (ptr != NULL) { 1259 assert(malloc_initialized || IS_INITIALIZER); 1260 malloc_thread_init(); 1261 1262 if ((config_prof && opt_prof) || config_stats || 1263 (config_valgrind && in_valgrind)) 1264 old_usize = isalloc(ptr, config_prof); 1265 if (config_valgrind && in_valgrind) 1266 old_rzsize = config_prof ? p2rz(ptr) : u2rz(old_usize); 1267 1268 if (config_prof && opt_prof) { 1269 prof_thr_cnt_t *cnt; 1270 1271 usize = s2u(size); 1272 PROF_ALLOC_PREP(usize, cnt); 1273 ret = irealloc_prof(ptr, old_usize, usize, cnt); 1274 } else { 1275 if (config_stats || (config_valgrind && in_valgrind)) 1276 usize = s2u(size); 1277 ret = iralloc(ptr, size, 0, 0, false); 1278 } 1279 } else { 1280 /* realloc(NULL, size) is equivalent to malloc(size). */ 1281 ret = imalloc_body(size, &usize); 1282 } 1283 1284 if (ret == NULL) { 1285 if (config_xmalloc && opt_xmalloc) { 1286 malloc_write("<jemalloc>: Error in realloc(): " 1287 "out of memory\n"); 1288 abort(); 1289 } 1290 set_errno(ENOMEM); 1291 } 1292 if (config_stats && ret != NULL) { 1293 thread_allocated_t *ta; 1294 assert(usize == isalloc(ret, config_prof)); 1295 ta = thread_allocated_tsd_get(); 1296 ta->allocated += usize; 1297 ta->deallocated += old_usize; 1298 } 1299 UTRACE(ptr, size, ret); 1300 JEMALLOC_VALGRIND_REALLOC(true, ret, usize, true, ptr, old_usize, 1301 old_rzsize, true, false); 1302 return (ret); 1303} 1304 1305void 1306je_free(void *ptr) 1307{ 1308 1309 UTRACE(ptr, 0, 0); 1310 if (ptr != NULL) 1311 ifree(ptr); 1312} 1313 1314/* 1315 * End malloc(3)-compatible functions. 1316 */ 1317/******************************************************************************/ 1318/* 1319 * Begin non-standard override functions. 1320 */ 1321 1322#ifdef JEMALLOC_OVERRIDE_MEMALIGN 1323void * 1324je_memalign(size_t alignment, size_t size) 1325{ 1326 void *ret JEMALLOC_CC_SILENCE_INIT(NULL); 1327 imemalign(&ret, alignment, size, 1); 1328 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); 1329 return (ret); 1330} 1331#endif 1332 1333#ifdef JEMALLOC_OVERRIDE_VALLOC 1334void * 1335je_valloc(size_t size) 1336{ 1337 void *ret JEMALLOC_CC_SILENCE_INIT(NULL); 1338 imemalign(&ret, PAGE, size, 1); 1339 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); 1340 return (ret); 1341} 1342#endif 1343 1344/* 1345 * is_malloc(je_malloc) is some macro magic to detect if jemalloc_defs.h has 1346 * #define je_malloc malloc 1347 */ 1348#define malloc_is_malloc 1 1349#define is_malloc_(a) malloc_is_ ## a 1350#define is_malloc(a) is_malloc_(a) 1351 1352#if ((is_malloc(je_malloc) == 1) && defined(__GLIBC__) && !defined(__UCLIBC__)) 1353/* 1354 * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible 1355 * to inconsistently reference libc's malloc(3)-compatible functions 1356 * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541). 1357 * 1358 * These definitions interpose hooks in glibc. The functions are actually 1359 * passed an extra argument for the caller return address, which will be 1360 * ignored. 1361 */ 1362JEMALLOC_EXPORT void (*__free_hook)(void *ptr) = je_free; 1363JEMALLOC_EXPORT void *(*__malloc_hook)(size_t size) = je_malloc; 1364JEMALLOC_EXPORT void *(*__realloc_hook)(void *ptr, size_t size) = je_realloc; 1365JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) = 1366 je_memalign; 1367#endif 1368 1369/* 1370 * End non-standard override functions. 1371 */ 1372/******************************************************************************/ 1373/* 1374 * Begin non-standard functions. 1375 */ 1376 1377JEMALLOC_ALWAYS_INLINE_C void * 1378imallocx(size_t usize, size_t alignment, bool zero, bool try_tcache, 1379 arena_t *arena) 1380{ 1381 1382 assert(usize == ((alignment == 0) ? s2u(usize) : sa2u(usize, 1383 alignment))); 1384 1385 if (alignment != 0) 1386 return (ipalloct(usize, alignment, zero, try_tcache, arena)); 1387 else if (zero) 1388 return (icalloct(usize, try_tcache, arena)); 1389 else 1390 return (imalloct(usize, try_tcache, arena)); 1391} 1392 1393static void * 1394imallocx_prof_sample(size_t usize, size_t alignment, bool zero, bool try_tcache, 1395 arena_t *arena, prof_thr_cnt_t *cnt) 1396{ 1397 void *p; 1398 1399 if (cnt == NULL) 1400 return (NULL); 1401 if (usize <= SMALL_MAXCLASS) { 1402 size_t usize_promoted = (alignment == 0) ? 1403 s2u(SMALL_MAXCLASS+1) : sa2u(SMALL_MAXCLASS+1, alignment); 1404 assert(usize_promoted != 0); 1405 p = imallocx(usize_promoted, alignment, zero, try_tcache, 1406 arena); 1407 if (p == NULL) 1408 return (NULL); 1409 arena_prof_promoted(p, usize); 1410 } else 1411 p = imallocx(usize, alignment, zero, try_tcache, arena); 1412 1413 return (p); 1414} 1415 1416JEMALLOC_ALWAYS_INLINE_C void * 1417imallocx_prof(size_t usize, size_t alignment, bool zero, bool try_tcache, 1418 arena_t *arena, prof_thr_cnt_t *cnt) 1419{ 1420 void *p; 1421 1422 if ((uintptr_t)cnt != (uintptr_t)1U) { 1423 p = imallocx_prof_sample(usize, alignment, zero, try_tcache, 1424 arena, cnt); 1425 } else 1426 p = imallocx(usize, alignment, zero, try_tcache, arena); 1427 if (p == NULL) 1428 return (NULL); 1429 prof_malloc(p, usize, cnt); 1430 1431 return (p); 1432} 1433 1434void * 1435je_mallocx(size_t size, int flags) 1436{ 1437 void *p; 1438 size_t usize; 1439 size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK) 1440 & (SIZE_T_MAX-1)); 1441 bool zero = flags & MALLOCX_ZERO; 1442 unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; 1443 arena_t *arena; 1444 bool try_tcache; 1445 1446 assert(size != 0); 1447 1448 if (malloc_init()) 1449 goto label_oom; 1450 1451 if (arena_ind != UINT_MAX) { 1452 arena = arenas[arena_ind]; 1453 try_tcache = false; 1454 } else { 1455 arena = NULL; 1456 try_tcache = true; 1457 } 1458 1459 usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); 1460 assert(usize != 0); 1461 1462 if (config_prof && opt_prof) { 1463 prof_thr_cnt_t *cnt; 1464 1465 PROF_ALLOC_PREP(usize, cnt); 1466 p = imallocx_prof(usize, alignment, zero, try_tcache, arena, 1467 cnt); 1468 } else 1469 p = imallocx(usize, alignment, zero, try_tcache, arena); 1470 if (p == NULL) 1471 goto label_oom; 1472 1473 if (config_stats) { 1474 assert(usize == isalloc(p, config_prof)); 1475 thread_allocated_tsd_get()->allocated += usize; 1476 } 1477 UTRACE(0, size, p); 1478 JEMALLOC_VALGRIND_MALLOC(true, p, usize, zero); 1479 return (p); 1480label_oom: 1481 if (config_xmalloc && opt_xmalloc) { 1482 malloc_write("<jemalloc>: Error in mallocx(): out of memory\n"); 1483 abort(); 1484 } 1485 UTRACE(0, size, 0); 1486 return (NULL); 1487} 1488 1489static void * 1490irallocx_prof_sample(void *oldptr, size_t size, size_t alignment, size_t usize, 1491 bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena, 1492 prof_thr_cnt_t *cnt) 1493{ 1494 void *p; 1495 1496 if (cnt == NULL) 1497 return (NULL); 1498 if (usize <= SMALL_MAXCLASS) { 1499 p = iralloct(oldptr, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >= 1500 size) ? 0 : size - (SMALL_MAXCLASS+1), alignment, zero, 1501 try_tcache_alloc, try_tcache_dalloc, arena); 1502 if (p == NULL) 1503 return (NULL); 1504 arena_prof_promoted(p, usize); 1505 } else { 1506 p = iralloct(oldptr, size, 0, alignment, zero, 1507 try_tcache_alloc, try_tcache_dalloc, arena); 1508 } 1509 1510 return (p); 1511} 1512 1513JEMALLOC_ALWAYS_INLINE_C void * 1514irallocx_prof(void *oldptr, size_t old_usize, size_t size, size_t alignment, 1515 size_t *usize, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, 1516 arena_t *arena, prof_thr_cnt_t *cnt) 1517{ 1518 void *p; 1519 prof_ctx_t *old_ctx; 1520 1521 old_ctx = prof_ctx_get(oldptr); 1522 if ((uintptr_t)cnt != (uintptr_t)1U) 1523 p = irallocx_prof_sample(oldptr, size, alignment, *usize, zero, 1524 try_tcache_alloc, try_tcache_dalloc, arena, cnt); 1525 else { 1526 p = iralloct(oldptr, size, 0, alignment, zero, 1527 try_tcache_alloc, try_tcache_dalloc, arena); 1528 } 1529 if (p == NULL) 1530 return (NULL); 1531 1532 if (p == oldptr && alignment != 0) { 1533 /* 1534 * The allocation did not move, so it is possible that the size 1535 * class is smaller than would guarantee the requested 1536 * alignment, and that the alignment constraint was 1537 * serendipitously satisfied. Additionally, old_usize may not 1538 * be the same as the current usize because of in-place large 1539 * reallocation. Therefore, query the actual value of usize. 1540 */ 1541 *usize = isalloc(p, config_prof); 1542 } 1543 prof_realloc(p, *usize, cnt, old_usize, old_ctx); 1544 1545 return (p); 1546} 1547 1548void * 1549je_rallocx(void *ptr, size_t size, int flags) 1550{ 1551 void *p; 1552 size_t usize, old_usize; 1553 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 1554 size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK) 1555 & (SIZE_T_MAX-1)); 1556 bool zero = flags & MALLOCX_ZERO; 1557 unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; 1558 bool try_tcache_alloc, try_tcache_dalloc; 1559 arena_t *arena; 1560 1561 assert(ptr != NULL); 1562 assert(size != 0); 1563 assert(malloc_initialized || IS_INITIALIZER); 1564 malloc_thread_init(); 1565 1566 if (arena_ind != UINT_MAX) { 1567 arena_chunk_t *chunk; 1568 try_tcache_alloc = false; 1569 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1570 try_tcache_dalloc = (chunk == ptr || chunk->arena != 1571 arenas[arena_ind]); 1572 arena = arenas[arena_ind]; 1573 } else { 1574 try_tcache_alloc = true; 1575 try_tcache_dalloc = true; 1576 arena = NULL; 1577 } 1578 1579 if ((config_prof && opt_prof) || config_stats || 1580 (config_valgrind && in_valgrind)) 1581 old_usize = isalloc(ptr, config_prof); 1582 if (config_valgrind && in_valgrind) 1583 old_rzsize = u2rz(old_usize); 1584 1585 if (config_prof && opt_prof) { 1586 prof_thr_cnt_t *cnt; 1587 1588 usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); 1589 assert(usize != 0); 1590 PROF_ALLOC_PREP(usize, cnt); 1591 p = irallocx_prof(ptr, old_usize, size, alignment, &usize, zero, 1592 try_tcache_alloc, try_tcache_dalloc, arena, cnt); 1593 if (p == NULL) 1594 goto label_oom; 1595 } else { 1596 p = iralloct(ptr, size, 0, alignment, zero, try_tcache_alloc, 1597 try_tcache_dalloc, arena); 1598 if (p == NULL) 1599 goto label_oom; 1600 if (config_stats || (config_valgrind && in_valgrind)) 1601 usize = isalloc(p, config_prof); 1602 } 1603 1604 if (config_stats) { 1605 thread_allocated_t *ta; 1606 ta = thread_allocated_tsd_get(); 1607 ta->allocated += usize; 1608 ta->deallocated += old_usize; 1609 } 1610 UTRACE(ptr, size, p); 1611 JEMALLOC_VALGRIND_REALLOC(true, p, usize, false, ptr, old_usize, 1612 old_rzsize, false, zero); 1613 return (p); 1614label_oom: 1615 if (config_xmalloc && opt_xmalloc) { 1616 malloc_write("<jemalloc>: Error in rallocx(): out of memory\n"); 1617 abort(); 1618 } 1619 UTRACE(ptr, size, 0); 1620 return (NULL); 1621} 1622 1623JEMALLOC_ALWAYS_INLINE_C size_t 1624ixallocx_helper(void *ptr, size_t old_usize, size_t size, size_t extra, 1625 size_t alignment, bool zero, arena_t *arena) 1626{ 1627 size_t usize; 1628 1629 if (ixalloc(ptr, size, extra, alignment, zero)) 1630 return (old_usize); 1631 usize = isalloc(ptr, config_prof); 1632 1633 return (usize); 1634} 1635 1636static size_t 1637ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra, 1638 size_t alignment, size_t max_usize, bool zero, arena_t *arena, 1639 prof_thr_cnt_t *cnt) 1640{ 1641 size_t usize; 1642 1643 if (cnt == NULL) 1644 return (old_usize); 1645 /* Use minimum usize to determine whether promotion may happen. */ 1646 if (((alignment == 0) ? s2u(size) : sa2u(size, alignment)) <= 1647 SMALL_MAXCLASS) { 1648 if (ixalloc(ptr, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >= 1649 size+extra) ? 0 : size+extra - (SMALL_MAXCLASS+1), 1650 alignment, zero)) 1651 return (old_usize); 1652 usize = isalloc(ptr, config_prof); 1653 if (max_usize < PAGE) 1654 arena_prof_promoted(ptr, usize); 1655 } else { 1656 usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, 1657 zero, arena); 1658 } 1659 1660 return (usize); 1661} 1662 1663JEMALLOC_ALWAYS_INLINE_C size_t 1664ixallocx_prof(void *ptr, size_t old_usize, size_t size, size_t extra, 1665 size_t alignment, size_t max_usize, bool zero, arena_t *arena, 1666 prof_thr_cnt_t *cnt) 1667{ 1668 size_t usize; 1669 prof_ctx_t *old_ctx; 1670 1671 old_ctx = prof_ctx_get(ptr); 1672 if ((uintptr_t)cnt != (uintptr_t)1U) { 1673 usize = ixallocx_prof_sample(ptr, old_usize, size, extra, 1674 alignment, zero, max_usize, arena, cnt); 1675 } else { 1676 usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, 1677 zero, arena); 1678 } 1679 if (usize == old_usize) 1680 return (usize); 1681 prof_realloc(ptr, usize, cnt, old_usize, old_ctx); 1682 1683 return (usize); 1684} 1685 1686size_t 1687je_xallocx(void *ptr, size_t size, size_t extra, int flags) 1688{ 1689 size_t usize, old_usize; 1690 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 1691 size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK) 1692 & (SIZE_T_MAX-1)); 1693 bool zero = flags & MALLOCX_ZERO; 1694 unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; 1695 arena_t *arena; 1696 1697 assert(ptr != NULL); 1698 assert(size != 0); 1699 assert(SIZE_T_MAX - size >= extra); 1700 assert(malloc_initialized || IS_INITIALIZER); 1701 malloc_thread_init(); 1702 1703 if (arena_ind != UINT_MAX) 1704 arena = arenas[arena_ind]; 1705 else 1706 arena = NULL; 1707 1708 old_usize = isalloc(ptr, config_prof); 1709 if (config_valgrind && in_valgrind) 1710 old_rzsize = u2rz(old_usize); 1711 1712 if (config_prof && opt_prof) { 1713 prof_thr_cnt_t *cnt; 1714 /* 1715 * usize isn't knowable before ixalloc() returns when extra is 1716 * non-zero. Therefore, compute its maximum possible value and 1717 * use that in PROF_ALLOC_PREP() to decide whether to capture a 1718 * backtrace. prof_realloc() will use the actual usize to 1719 * decide whether to sample. 1720 */ 1721 size_t max_usize = (alignment == 0) ? s2u(size+extra) : 1722 sa2u(size+extra, alignment); 1723 PROF_ALLOC_PREP(max_usize, cnt); 1724 usize = ixallocx_prof(ptr, old_usize, size, extra, alignment, 1725 max_usize, zero, arena, cnt); 1726 } else { 1727 usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, 1728 zero, arena); 1729 } 1730 if (usize == old_usize) 1731 goto label_not_resized; 1732 1733 if (config_stats) { 1734 thread_allocated_t *ta; 1735 ta = thread_allocated_tsd_get(); 1736 ta->allocated += usize; 1737 ta->deallocated += old_usize; 1738 } 1739 JEMALLOC_VALGRIND_REALLOC(false, ptr, usize, false, ptr, old_usize, 1740 old_rzsize, false, zero); 1741label_not_resized: 1742 UTRACE(ptr, size, ptr); 1743 return (usize); 1744} 1745 1746size_t 1747je_sallocx(const void *ptr, int flags) 1748{ 1749 size_t usize; 1750 1751 assert(malloc_initialized || IS_INITIALIZER); 1752 malloc_thread_init(); 1753 1754 if (config_ivsalloc) 1755 usize = ivsalloc(ptr, config_prof); 1756 else { 1757 assert(ptr != NULL); 1758 usize = isalloc(ptr, config_prof); 1759 } 1760 1761 return (usize); 1762} 1763 1764void 1765je_dallocx(void *ptr, int flags) 1766{ 1767 size_t usize; 1768 UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); 1769 unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; 1770 bool try_tcache; 1771 1772 assert(ptr != NULL); 1773 assert(malloc_initialized || IS_INITIALIZER); 1774 1775 if (arena_ind != UINT_MAX) { 1776 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1777 try_tcache = (chunk == ptr || chunk->arena != 1778 arenas[arena_ind]); 1779 } else 1780 try_tcache = true; 1781 1782 UTRACE(ptr, 0, 0); 1783 if (config_stats || config_valgrind) 1784 usize = isalloc(ptr, config_prof); 1785 if (config_prof && opt_prof) { 1786 if (config_stats == false && config_valgrind == false) 1787 usize = isalloc(ptr, config_prof); 1788 prof_free(ptr, usize); 1789 } 1790 if (config_stats) 1791 thread_allocated_tsd_get()->deallocated += usize; 1792 if (config_valgrind && in_valgrind) 1793 rzsize = p2rz(ptr); 1794 iqalloct(ptr, try_tcache); 1795 JEMALLOC_VALGRIND_FREE(ptr, rzsize); 1796} 1797 1798size_t 1799je_nallocx(size_t size, int flags) 1800{ 1801 size_t usize; 1802 size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK) 1803 & (SIZE_T_MAX-1)); 1804 1805 assert(size != 0); 1806 1807 if (malloc_init()) 1808 return (0); 1809 1810 usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); 1811 assert(usize != 0); 1812 return (usize); 1813} 1814 1815int 1816je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, 1817 size_t newlen) 1818{ 1819 1820 if (malloc_init()) 1821 return (EAGAIN); 1822 1823 return (ctl_byname(name, oldp, oldlenp, newp, newlen)); 1824} 1825 1826int 1827je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) 1828{ 1829 1830 if (malloc_init()) 1831 return (EAGAIN); 1832 1833 return (ctl_nametomib(name, mibp, miblenp)); 1834} 1835 1836int 1837je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, 1838 void *newp, size_t newlen) 1839{ 1840 1841 if (malloc_init()) 1842 return (EAGAIN); 1843 1844 return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen)); 1845} 1846 1847void 1848je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, 1849 const char *opts) 1850{ 1851 1852 stats_print(write_cb, cbopaque, opts); 1853} 1854 1855size_t 1856je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) 1857{ 1858 size_t ret; 1859 1860 assert(malloc_initialized || IS_INITIALIZER); 1861 malloc_thread_init(); 1862 1863 if (config_ivsalloc) 1864 ret = ivsalloc(ptr, config_prof); 1865 else 1866 ret = (ptr != NULL) ? isalloc(ptr, config_prof) : 0; 1867 1868 return (ret); 1869} 1870 1871/* 1872 * End non-standard functions. 1873 */ 1874/******************************************************************************/ 1875/* 1876 * The following functions are used by threading libraries for protection of 1877 * malloc during fork(). 1878 */ 1879 1880/* 1881 * If an application creates a thread before doing any allocation in the main 1882 * thread, then calls fork(2) in the main thread followed by memory allocation 1883 * in the child process, a race can occur that results in deadlock within the 1884 * child: the main thread may have forked while the created thread had 1885 * partially initialized the allocator. Ordinarily jemalloc prevents 1886 * fork/malloc races via the following functions it registers during 1887 * initialization using pthread_atfork(), but of course that does no good if 1888 * the allocator isn't fully initialized at fork time. The following library 1889 * constructor is a partial solution to this problem. It may still possible to 1890 * trigger the deadlock described above, but doing so would involve forking via 1891 * a library constructor that runs before jemalloc's runs. 1892 */ 1893JEMALLOC_ATTR(constructor) 1894static void 1895jemalloc_constructor(void) 1896{ 1897 1898 malloc_init(); 1899} 1900 1901#ifndef JEMALLOC_MUTEX_INIT_CB 1902void 1903jemalloc_prefork(void) 1904#else 1905JEMALLOC_EXPORT void 1906_malloc_prefork(void) 1907#endif 1908{ 1909 unsigned i; 1910 1911#ifdef JEMALLOC_MUTEX_INIT_CB 1912 if (malloc_initialized == false) 1913 return; 1914#endif 1915 assert(malloc_initialized); 1916 1917 /* Acquire all mutexes in a safe order. */ 1918 ctl_prefork(); 1919 prof_prefork(); 1920 malloc_mutex_prefork(&arenas_lock); 1921 for (i = 0; i < narenas_total; i++) { 1922 if (arenas[i] != NULL) 1923 arena_prefork(arenas[i]); 1924 } 1925 chunk_prefork(); 1926 base_prefork(); 1927 huge_prefork(); 1928} 1929 1930#ifndef JEMALLOC_MUTEX_INIT_CB 1931void 1932jemalloc_postfork_parent(void) 1933#else 1934JEMALLOC_EXPORT void 1935_malloc_postfork(void) 1936#endif 1937{ 1938 unsigned i; 1939 1940#ifdef JEMALLOC_MUTEX_INIT_CB 1941 if (malloc_initialized == false) 1942 return; 1943#endif 1944 assert(malloc_initialized); 1945 1946 /* Release all mutexes, now that fork() has completed. */ 1947 huge_postfork_parent(); 1948 base_postfork_parent(); 1949 chunk_postfork_parent(); 1950 for (i = 0; i < narenas_total; i++) { 1951 if (arenas[i] != NULL) 1952 arena_postfork_parent(arenas[i]); 1953 } 1954 malloc_mutex_postfork_parent(&arenas_lock); 1955 prof_postfork_parent(); 1956 ctl_postfork_parent(); 1957} 1958 1959void 1960jemalloc_postfork_child(void) 1961{ 1962 unsigned i; 1963 1964 assert(malloc_initialized); 1965 1966 /* Release all mutexes, now that fork() has completed. */ 1967 huge_postfork_child(); 1968 base_postfork_child(); 1969 chunk_postfork_child(); 1970 for (i = 0; i < narenas_total; i++) { 1971 if (arenas[i] != NULL) 1972 arena_postfork_child(arenas[i]); 1973 } 1974 malloc_mutex_postfork_child(&arenas_lock); 1975 prof_postfork_child(); 1976 ctl_postfork_child(); 1977} 1978 1979/******************************************************************************/ 1980/* 1981 * The following functions are used for TLS allocation/deallocation in static 1982 * binaries on FreeBSD. The primary difference between these and i[mcd]alloc() 1983 * is that these avoid accessing TLS variables. 1984 */ 1985 1986static void * 1987a0alloc(size_t size, bool zero) 1988{ 1989 1990 if (malloc_init()) 1991 return (NULL); 1992 1993 if (size == 0) 1994 size = 1; 1995 1996 if (size <= arena_maxclass) 1997 return (arena_malloc(arenas[0], size, zero, false)); 1998 else 1999 return (huge_malloc(NULL, size, zero)); 2000} 2001 2002void * 2003a0malloc(size_t size) 2004{ 2005 2006 return (a0alloc(size, false)); 2007} 2008 2009void * 2010a0calloc(size_t num, size_t size) 2011{ 2012 2013 return (a0alloc(num * size, true)); 2014} 2015 2016void 2017a0free(void *ptr) 2018{ 2019 arena_chunk_t *chunk; 2020 2021 if (ptr == NULL) 2022 return; 2023 2024 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 2025 if (chunk != ptr) 2026 arena_dalloc(chunk, ptr, false); 2027 else 2028 huge_dalloc(ptr); 2029} 2030 2031/******************************************************************************/ 2032