jemalloc_internal.h.in revision 86abd0dcd8e478759fe409d338d11558c4cec427
1#ifndef JEMALLOC_INTERNAL_H 2#define JEMALLOC_INTERNAL_H 3#include <math.h> 4#ifdef _WIN32 5# include <windows.h> 6# define ENOENT ERROR_PATH_NOT_FOUND 7# define EINVAL ERROR_BAD_ARGUMENTS 8# define EAGAIN ERROR_OUTOFMEMORY 9# define EPERM ERROR_WRITE_FAULT 10# define EFAULT ERROR_INVALID_ADDRESS 11# define ENOMEM ERROR_NOT_ENOUGH_MEMORY 12# undef ERANGE 13# define ERANGE ERROR_INVALID_DATA 14#else 15# include <sys/param.h> 16# include <sys/mman.h> 17# include <sys/syscall.h> 18# if !defined(SYS_write) && defined(__NR_write) 19# define SYS_write __NR_write 20# endif 21# include <sys/uio.h> 22# include <pthread.h> 23# include <errno.h> 24#endif 25#include <sys/types.h> 26 27#include <limits.h> 28#ifndef SIZE_T_MAX 29# define SIZE_T_MAX SIZE_MAX 30#endif 31#include <stdarg.h> 32#include <stdbool.h> 33#include <stdio.h> 34#include <stdlib.h> 35#include <stdint.h> 36#include <stddef.h> 37#ifndef offsetof 38# define offsetof(type, member) ((size_t)&(((type *)NULL)->member)) 39#endif 40#include <inttypes.h> 41#include <string.h> 42#include <strings.h> 43#include <ctype.h> 44#ifdef _MSC_VER 45# include <io.h> 46typedef intptr_t ssize_t; 47# define PATH_MAX 1024 48# define STDERR_FILENO 2 49# define __func__ __FUNCTION__ 50/* Disable warnings about deprecated system functions */ 51# pragma warning(disable: 4996) 52#else 53# include <unistd.h> 54#endif 55#include <fcntl.h> 56 57#include "jemalloc_internal_defs.h" 58 59#ifdef JEMALLOC_UTRACE 60#include <sys/ktrace.h> 61#endif 62 63#ifdef JEMALLOC_VALGRIND 64#include <valgrind/valgrind.h> 65#include <valgrind/memcheck.h> 66#endif 67 68#define JEMALLOC_NO_DEMANGLE 69#ifdef JEMALLOC_JET 70# define JEMALLOC_N(n) jet_##n 71# include "jemalloc/internal/public_namespace.h" 72# define JEMALLOC_NO_RENAME 73# include "../jemalloc@install_suffix@.h" 74#else 75# define JEMALLOC_N(n) @private_namespace@##n 76# include "../jemalloc@install_suffix@.h" 77#endif 78#include "jemalloc/internal/private_namespace.h" 79 80#ifdef JEMALLOC_CC_SILENCE 81#define UNUSED JEMALLOC_ATTR(unused) 82#else 83#define UNUSED 84#endif 85 86static const bool config_debug = 87#ifdef JEMALLOC_DEBUG 88 true 89#else 90 false 91#endif 92 ; 93static const bool config_dss = 94#ifdef JEMALLOC_DSS 95 true 96#else 97 false 98#endif 99 ; 100static const bool config_fill = 101#ifdef JEMALLOC_FILL 102 true 103#else 104 false 105#endif 106 ; 107static const bool config_lazy_lock = 108#ifdef JEMALLOC_LAZY_LOCK 109 true 110#else 111 false 112#endif 113 ; 114static const bool config_prof = 115#ifdef JEMALLOC_PROF 116 true 117#else 118 false 119#endif 120 ; 121static const bool config_prof_libgcc = 122#ifdef JEMALLOC_PROF_LIBGCC 123 true 124#else 125 false 126#endif 127 ; 128static const bool config_prof_libunwind = 129#ifdef JEMALLOC_PROF_LIBUNWIND 130 true 131#else 132 false 133#endif 134 ; 135static const bool config_mremap = 136#ifdef JEMALLOC_MREMAP 137 true 138#else 139 false 140#endif 141 ; 142static const bool config_munmap = 143#ifdef JEMALLOC_MUNMAP 144 true 145#else 146 false 147#endif 148 ; 149static const bool config_stats = 150#ifdef JEMALLOC_STATS 151 true 152#else 153 false 154#endif 155 ; 156static const bool config_tcache = 157#ifdef JEMALLOC_TCACHE 158 true 159#else 160 false 161#endif 162 ; 163static const bool config_tls = 164#ifdef JEMALLOC_TLS 165 true 166#else 167 false 168#endif 169 ; 170static const bool config_utrace = 171#ifdef JEMALLOC_UTRACE 172 true 173#else 174 false 175#endif 176 ; 177static const bool config_valgrind = 178#ifdef JEMALLOC_VALGRIND 179 true 180#else 181 false 182#endif 183 ; 184static const bool config_xmalloc = 185#ifdef JEMALLOC_XMALLOC 186 true 187#else 188 false 189#endif 190 ; 191static const bool config_ivsalloc = 192#ifdef JEMALLOC_IVSALLOC 193 true 194#else 195 false 196#endif 197 ; 198 199#ifdef JEMALLOC_ATOMIC9 200#include <machine/atomic.h> 201#endif 202 203#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN)) 204#include <libkern/OSAtomic.h> 205#endif 206 207#ifdef JEMALLOC_ZONE 208#include <mach/mach_error.h> 209#include <mach/mach_init.h> 210#include <mach/vm_map.h> 211#include <malloc/malloc.h> 212#endif 213 214#define RB_COMPACT 215#include "jemalloc/internal/rb.h" 216#include "jemalloc/internal/qr.h" 217#include "jemalloc/internal/ql.h" 218 219/* 220 * jemalloc can conceptually be broken into components (arena, tcache, etc.), 221 * but there are circular dependencies that cannot be broken without 222 * substantial performance degradation. In order to reduce the effect on 223 * visual code flow, read the header files in multiple passes, with one of the 224 * following cpp variables defined during each pass: 225 * 226 * JEMALLOC_H_TYPES : Preprocessor-defined constants and psuedo-opaque data 227 * types. 228 * JEMALLOC_H_STRUCTS : Data structures. 229 * JEMALLOC_H_EXTERNS : Extern data declarations and function prototypes. 230 * JEMALLOC_H_INLINES : Inline functions. 231 */ 232/******************************************************************************/ 233#define JEMALLOC_H_TYPES 234 235#define ALLOCM_LG_ALIGN_MASK ((int)0x3f) 236 237#define ZU(z) ((size_t)z) 238#define QU(q) ((uint64_t)q) 239 240#ifndef __DECONST 241# define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var)) 242#endif 243 244/* 245 * JEMALLOC_ALWAYS_INLINE is used within header files for functions that are 246 * static inline functions if inlining is enabled, and single-definition 247 * library-private functions if inlining is disabled. 248 * 249 * JEMALLOC_ALWAYS_INLINE_C is for use in .c files, in which case the denoted 250 * functions are always static, regardless of whether inlining is enabled. 251 */ 252#ifdef JEMALLOC_DEBUG 253 /* Disable inlining to make debugging easier. */ 254# define JEMALLOC_ALWAYS_INLINE 255# define JEMALLOC_ALWAYS_INLINE_C static 256# define JEMALLOC_INLINE 257# define inline 258#else 259# define JEMALLOC_ENABLE_INLINE 260# ifdef JEMALLOC_HAVE_ATTR 261# define JEMALLOC_ALWAYS_INLINE \ 262 static inline JEMALLOC_ATTR(unused) JEMALLOC_ATTR(always_inline) 263# define JEMALLOC_ALWAYS_INLINE_C \ 264 static inline JEMALLOC_ATTR(always_inline) 265# else 266# define JEMALLOC_ALWAYS_INLINE static inline 267# define JEMALLOC_ALWAYS_INLINE_C static inline 268# endif 269# define JEMALLOC_INLINE static inline 270# ifdef _MSC_VER 271# define inline _inline 272# endif 273#endif 274 275/* Smallest size class to support. */ 276#define LG_TINY_MIN 3 277#define TINY_MIN (1U << LG_TINY_MIN) 278 279/* 280 * Minimum alignment of allocations is 2^LG_QUANTUM bytes (ignoring tiny size 281 * classes). 282 */ 283#ifndef LG_QUANTUM 284# if (defined(__i386__) || defined(_M_IX86)) 285# define LG_QUANTUM 4 286# endif 287# ifdef __ia64__ 288# define LG_QUANTUM 4 289# endif 290# ifdef __alpha__ 291# define LG_QUANTUM 4 292# endif 293# ifdef __sparc64__ 294# define LG_QUANTUM 4 295# endif 296# if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64)) 297# define LG_QUANTUM 4 298# endif 299# ifdef __arm__ 300# define LG_QUANTUM 3 301# endif 302# ifdef __aarch64__ 303# define LG_QUANTUM 4 304# endif 305# ifdef __hppa__ 306# define LG_QUANTUM 4 307# endif 308# ifdef __mips__ 309# define LG_QUANTUM 3 310# endif 311# ifdef __powerpc__ 312# define LG_QUANTUM 4 313# endif 314# ifdef __s390__ 315# define LG_QUANTUM 4 316# endif 317# ifdef __SH4__ 318# define LG_QUANTUM 4 319# endif 320# ifdef __tile__ 321# define LG_QUANTUM 4 322# endif 323# ifndef LG_QUANTUM 324# error "No LG_QUANTUM definition for architecture; specify via CPPFLAGS" 325# endif 326#endif 327 328#define QUANTUM ((size_t)(1U << LG_QUANTUM)) 329#define QUANTUM_MASK (QUANTUM - 1) 330 331/* Return the smallest quantum multiple that is >= a. */ 332#define QUANTUM_CEILING(a) \ 333 (((a) + QUANTUM_MASK) & ~QUANTUM_MASK) 334 335#define LONG ((size_t)(1U << LG_SIZEOF_LONG)) 336#define LONG_MASK (LONG - 1) 337 338/* Return the smallest long multiple that is >= a. */ 339#define LONG_CEILING(a) \ 340 (((a) + LONG_MASK) & ~LONG_MASK) 341 342#define SIZEOF_PTR (1U << LG_SIZEOF_PTR) 343#define PTR_MASK (SIZEOF_PTR - 1) 344 345/* Return the smallest (void *) multiple that is >= a. */ 346#define PTR_CEILING(a) \ 347 (((a) + PTR_MASK) & ~PTR_MASK) 348 349/* 350 * Maximum size of L1 cache line. This is used to avoid cache line aliasing. 351 * In addition, this controls the spacing of cacheline-spaced size classes. 352 * 353 * CACHELINE cannot be based on LG_CACHELINE because __declspec(align()) can 354 * only handle raw constants. 355 */ 356#define LG_CACHELINE 6 357#define CACHELINE 64 358#define CACHELINE_MASK (CACHELINE - 1) 359 360/* Return the smallest cacheline multiple that is >= s. */ 361#define CACHELINE_CEILING(s) \ 362 (((s) + CACHELINE_MASK) & ~CACHELINE_MASK) 363 364/* Page size. STATIC_PAGE_SHIFT is determined by the configure script. */ 365#ifdef PAGE_MASK 366# undef PAGE_MASK 367#endif 368#define LG_PAGE STATIC_PAGE_SHIFT 369#define PAGE ((size_t)(1U << STATIC_PAGE_SHIFT)) 370#define PAGE_MASK ((size_t)(PAGE - 1)) 371 372/* Return the smallest pagesize multiple that is >= s. */ 373#define PAGE_CEILING(s) \ 374 (((s) + PAGE_MASK) & ~PAGE_MASK) 375 376/* Return the nearest aligned address at or below a. */ 377#define ALIGNMENT_ADDR2BASE(a, alignment) \ 378 ((void *)((uintptr_t)(a) & (-(alignment)))) 379 380/* Return the offset between a and the nearest aligned address at or below a. */ 381#define ALIGNMENT_ADDR2OFFSET(a, alignment) \ 382 ((size_t)((uintptr_t)(a) & (alignment - 1))) 383 384/* Return the smallest alignment multiple that is >= s. */ 385#define ALIGNMENT_CEILING(s, alignment) \ 386 (((s) + (alignment - 1)) & (-(alignment))) 387 388/* Declare a variable length array */ 389#if __STDC_VERSION__ < 199901L 390# ifdef _MSC_VER 391# include <malloc.h> 392# define alloca _alloca 393# else 394# ifdef JEMALLOC_HAS_ALLOCA_H 395# include <alloca.h> 396# else 397# include <stdlib.h> 398# endif 399# endif 400# define VARIABLE_ARRAY(type, name, count) \ 401 type *name = alloca(sizeof(type) * count) 402#else 403# define VARIABLE_ARRAY(type, name, count) type name[count] 404#endif 405 406#ifdef JEMALLOC_VALGRIND 407/* 408 * The JEMALLOC_VALGRIND_*() macros must be macros rather than functions 409 * so that when Valgrind reports errors, there are no extra stack frames 410 * in the backtraces. 411 * 412 * The size that is reported to valgrind must be consistent through a chain of 413 * malloc..realloc..realloc calls. Request size isn't recorded anywhere in 414 * jemalloc, so it is critical that all callers of these macros provide usize 415 * rather than request size. As a result, buffer overflow detection is 416 * technically weakened for the standard API, though it is generally accepted 417 * practice to consider any extra bytes reported by malloc_usable_size() as 418 * usable space. 419 */ 420#define JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero) do { \ 421 if (config_valgrind && opt_valgrind && cond) \ 422 VALGRIND_MALLOCLIKE_BLOCK(ptr, usize, p2rz(ptr), zero); \ 423} while (0) 424#define JEMALLOC_VALGRIND_REALLOC(ptr, usize, old_ptr, old_usize, \ 425 old_rzsize, zero) do { \ 426 if (config_valgrind && opt_valgrind) { \ 427 size_t rzsize = p2rz(ptr); \ 428 \ 429 if (ptr == old_ptr) { \ 430 VALGRIND_RESIZEINPLACE_BLOCK(ptr, old_usize, \ 431 usize, rzsize); \ 432 if (zero && old_usize < usize) { \ 433 VALGRIND_MAKE_MEM_DEFINED( \ 434 (void *)((uintptr_t)ptr + \ 435 old_usize), usize - old_usize); \ 436 } \ 437 } else { \ 438 if (old_ptr != NULL) { \ 439 VALGRIND_FREELIKE_BLOCK(old_ptr, \ 440 old_rzsize); \ 441 } \ 442 if (ptr != NULL) { \ 443 size_t copy_size = (old_usize < usize) \ 444 ? old_usize : usize; \ 445 size_t tail_size = usize - copy_size; \ 446 VALGRIND_MALLOCLIKE_BLOCK(ptr, usize, \ 447 rzsize, false); \ 448 if (copy_size > 0) { \ 449 VALGRIND_MAKE_MEM_DEFINED(ptr, \ 450 copy_size); \ 451 } \ 452 if (zero && tail_size > 0) { \ 453 VALGRIND_MAKE_MEM_DEFINED( \ 454 (void *)((uintptr_t)ptr + \ 455 copy_size), tail_size); \ 456 } \ 457 } \ 458 } \ 459 } \ 460} while (0) 461#define JEMALLOC_VALGRIND_FREE(ptr, rzsize) do { \ 462 if (config_valgrind && opt_valgrind) \ 463 VALGRIND_FREELIKE_BLOCK(ptr, rzsize); \ 464} while (0) 465#else 466#define RUNNING_ON_VALGRIND ((unsigned)0) 467#define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed) \ 468 do {} while (0) 469#define VALGRIND_RESIZEINPLACE_BLOCK(addr, oldSizeB, newSizeB, rzB) \ 470 do {} while (0) 471#define VALGRIND_FREELIKE_BLOCK(addr, rzB) do {} while (0) 472#define VALGRIND_MAKE_MEM_NOACCESS(_qzz_addr, _qzz_len) do {} while (0) 473#define VALGRIND_MAKE_MEM_UNDEFINED(_qzz_addr, _qzz_len) do {} while (0) 474#define VALGRIND_MAKE_MEM_DEFINED(_qzz_addr, _qzz_len) do {} while (0) 475#define JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero) do {} while (0) 476#define JEMALLOC_VALGRIND_REALLOC(ptr, usize, old_ptr, old_usize, \ 477 old_rzsize, zero) do {} while (0) 478#define JEMALLOC_VALGRIND_FREE(ptr, rzsize) do {} while (0) 479#endif 480 481#include "jemalloc/internal/util.h" 482#include "jemalloc/internal/atomic.h" 483#include "jemalloc/internal/prng.h" 484#include "jemalloc/internal/ckh.h" 485#include "jemalloc/internal/size_classes.h" 486#include "jemalloc/internal/stats.h" 487#include "jemalloc/internal/ctl.h" 488#include "jemalloc/internal/mutex.h" 489#include "jemalloc/internal/tsd.h" 490#include "jemalloc/internal/mb.h" 491#include "jemalloc/internal/extent.h" 492#include "jemalloc/internal/arena.h" 493#include "jemalloc/internal/bitmap.h" 494#include "jemalloc/internal/base.h" 495#include "jemalloc/internal/chunk.h" 496#include "jemalloc/internal/huge.h" 497#include "jemalloc/internal/rtree.h" 498#include "jemalloc/internal/tcache.h" 499#include "jemalloc/internal/hash.h" 500#include "jemalloc/internal/quarantine.h" 501#include "jemalloc/internal/prof.h" 502 503#undef JEMALLOC_H_TYPES 504/******************************************************************************/ 505#define JEMALLOC_H_STRUCTS 506 507#include "jemalloc/internal/util.h" 508#include "jemalloc/internal/atomic.h" 509#include "jemalloc/internal/prng.h" 510#include "jemalloc/internal/ckh.h" 511#include "jemalloc/internal/size_classes.h" 512#include "jemalloc/internal/stats.h" 513#include "jemalloc/internal/ctl.h" 514#include "jemalloc/internal/mutex.h" 515#include "jemalloc/internal/tsd.h" 516#include "jemalloc/internal/mb.h" 517#include "jemalloc/internal/bitmap.h" 518#include "jemalloc/internal/extent.h" 519#include "jemalloc/internal/arena.h" 520#include "jemalloc/internal/base.h" 521#include "jemalloc/internal/chunk.h" 522#include "jemalloc/internal/huge.h" 523#include "jemalloc/internal/rtree.h" 524#include "jemalloc/internal/tcache.h" 525#include "jemalloc/internal/hash.h" 526#include "jemalloc/internal/quarantine.h" 527#include "jemalloc/internal/prof.h" 528 529typedef struct { 530 uint64_t allocated; 531 uint64_t deallocated; 532} thread_allocated_t; 533/* 534 * The JEMALLOC_ARG_CONCAT() wrapper is necessary to pass {0, 0} via a cpp macro 535 * argument. 536 */ 537#define THREAD_ALLOCATED_INITIALIZER JEMALLOC_ARG_CONCAT({0, 0}) 538 539#undef JEMALLOC_H_STRUCTS 540/******************************************************************************/ 541#define JEMALLOC_H_EXTERNS 542 543extern bool opt_abort; 544extern bool opt_junk; 545extern size_t opt_quarantine; 546extern bool opt_redzone; 547extern bool opt_utrace; 548extern bool opt_valgrind; 549extern bool opt_xmalloc; 550extern bool opt_zero; 551extern size_t opt_narenas; 552 553/* Number of CPUs. */ 554extern unsigned ncpus; 555 556/* Protects arenas initialization (arenas, arenas_total). */ 557extern malloc_mutex_t arenas_lock; 558/* 559 * Arenas that are used to service external requests. Not all elements of the 560 * arenas array are necessarily used; arenas are created lazily as needed. 561 * 562 * arenas[0..narenas_auto) are used for automatic multiplexing of threads and 563 * arenas. arenas[narenas_auto..narenas_total) are only used if the application 564 * takes some action to create them and allocate from them. 565 */ 566extern arena_t **arenas; 567extern unsigned narenas_total; 568extern unsigned narenas_auto; /* Read-only after initialization. */ 569 570arena_t *arenas_extend(unsigned ind); 571void arenas_cleanup(void *arg); 572arena_t *choose_arena_hard(void); 573void jemalloc_prefork(void); 574void jemalloc_postfork_parent(void); 575void jemalloc_postfork_child(void); 576 577#include "jemalloc/internal/util.h" 578#include "jemalloc/internal/atomic.h" 579#include "jemalloc/internal/prng.h" 580#include "jemalloc/internal/ckh.h" 581#include "jemalloc/internal/size_classes.h" 582#include "jemalloc/internal/stats.h" 583#include "jemalloc/internal/ctl.h" 584#include "jemalloc/internal/mutex.h" 585#include "jemalloc/internal/tsd.h" 586#include "jemalloc/internal/mb.h" 587#include "jemalloc/internal/bitmap.h" 588#include "jemalloc/internal/extent.h" 589#include "jemalloc/internal/arena.h" 590#include "jemalloc/internal/base.h" 591#include "jemalloc/internal/chunk.h" 592#include "jemalloc/internal/huge.h" 593#include "jemalloc/internal/rtree.h" 594#include "jemalloc/internal/tcache.h" 595#include "jemalloc/internal/hash.h" 596#include "jemalloc/internal/quarantine.h" 597#include "jemalloc/internal/prof.h" 598 599#undef JEMALLOC_H_EXTERNS 600/******************************************************************************/ 601#define JEMALLOC_H_INLINES 602 603#include "jemalloc/internal/util.h" 604#include "jemalloc/internal/atomic.h" 605#include "jemalloc/internal/prng.h" 606#include "jemalloc/internal/ckh.h" 607#include "jemalloc/internal/size_classes.h" 608#include "jemalloc/internal/stats.h" 609#include "jemalloc/internal/ctl.h" 610#include "jemalloc/internal/mutex.h" 611#include "jemalloc/internal/tsd.h" 612#include "jemalloc/internal/mb.h" 613#include "jemalloc/internal/extent.h" 614#include "jemalloc/internal/base.h" 615#include "jemalloc/internal/chunk.h" 616#include "jemalloc/internal/huge.h" 617 618#ifndef JEMALLOC_ENABLE_INLINE 619malloc_tsd_protos(JEMALLOC_ATTR(unused), arenas, arena_t *) 620 621size_t s2u(size_t size); 622size_t sa2u(size_t size, size_t alignment); 623unsigned narenas_total_get(void); 624arena_t *choose_arena(arena_t *arena); 625#endif 626 627#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) 628/* 629 * Map of pthread_self() --> arenas[???], used for selecting an arena to use 630 * for allocations. 631 */ 632malloc_tsd_externs(arenas, arena_t *) 633malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, arenas, arena_t *, NULL, 634 arenas_cleanup) 635 636/* 637 * Compute usable size that would result from allocating an object with the 638 * specified size. 639 */ 640JEMALLOC_ALWAYS_INLINE size_t 641s2u(size_t size) 642{ 643 644 if (size <= SMALL_MAXCLASS) 645 return (arena_bin_info[SMALL_SIZE2BIN(size)].reg_size); 646 if (size <= arena_maxclass) 647 return (PAGE_CEILING(size)); 648 return (CHUNK_CEILING(size)); 649} 650 651/* 652 * Compute usable size that would result from allocating an object with the 653 * specified size and alignment. 654 */ 655JEMALLOC_ALWAYS_INLINE size_t 656sa2u(size_t size, size_t alignment) 657{ 658 size_t usize; 659 660 assert(alignment != 0 && ((alignment - 1) & alignment) == 0); 661 662 /* 663 * Round size up to the nearest multiple of alignment. 664 * 665 * This done, we can take advantage of the fact that for each small 666 * size class, every object is aligned at the smallest power of two 667 * that is non-zero in the base two representation of the size. For 668 * example: 669 * 670 * Size | Base 2 | Minimum alignment 671 * -----+----------+------------------ 672 * 96 | 1100000 | 32 673 * 144 | 10100000 | 32 674 * 192 | 11000000 | 64 675 */ 676 usize = ALIGNMENT_CEILING(size, alignment); 677 /* 678 * (usize < size) protects against the combination of maximal 679 * alignment and size greater than maximal alignment. 680 */ 681 if (usize < size) { 682 /* size_t overflow. */ 683 return (0); 684 } 685 686 if (usize <= arena_maxclass && alignment <= PAGE) { 687 if (usize <= SMALL_MAXCLASS) 688 return (arena_bin_info[SMALL_SIZE2BIN(usize)].reg_size); 689 return (PAGE_CEILING(usize)); 690 } else { 691 size_t run_size; 692 693 /* 694 * We can't achieve subpage alignment, so round up alignment 695 * permanently; it makes later calculations simpler. 696 */ 697 alignment = PAGE_CEILING(alignment); 698 usize = PAGE_CEILING(size); 699 /* 700 * (usize < size) protects against very large sizes within 701 * PAGE of SIZE_T_MAX. 702 * 703 * (usize + alignment < usize) protects against the 704 * combination of maximal alignment and usize large enough 705 * to cause overflow. This is similar to the first overflow 706 * check above, but it needs to be repeated due to the new 707 * usize value, which may now be *equal* to maximal 708 * alignment, whereas before we only detected overflow if the 709 * original size was *greater* than maximal alignment. 710 */ 711 if (usize < size || usize + alignment < usize) { 712 /* size_t overflow. */ 713 return (0); 714 } 715 716 /* 717 * Calculate the size of the over-size run that arena_palloc() 718 * would need to allocate in order to guarantee the alignment. 719 * If the run wouldn't fit within a chunk, round up to a huge 720 * allocation size. 721 */ 722 run_size = usize + alignment - PAGE; 723 if (run_size <= arena_maxclass) 724 return (PAGE_CEILING(usize)); 725 return (CHUNK_CEILING(usize)); 726 } 727} 728 729JEMALLOC_INLINE unsigned 730narenas_total_get(void) 731{ 732 unsigned narenas; 733 734 malloc_mutex_lock(&arenas_lock); 735 narenas = narenas_total; 736 malloc_mutex_unlock(&arenas_lock); 737 738 return (narenas); 739} 740 741/* Choose an arena based on a per-thread value. */ 742JEMALLOC_INLINE arena_t * 743choose_arena(arena_t *arena) 744{ 745 arena_t *ret; 746 747 if (arena != NULL) 748 return (arena); 749 750 if ((ret = *arenas_tsd_get()) == NULL) { 751 ret = choose_arena_hard(); 752 assert(ret != NULL); 753 } 754 755 return (ret); 756} 757#endif 758 759#include "jemalloc/internal/bitmap.h" 760#include "jemalloc/internal/rtree.h" 761/* 762 * Include arena.h twice in order to resolve circular dependencies with 763 * tcache.h. 764 */ 765#define JEMALLOC_ARENA_INLINE_A 766#include "jemalloc/internal/arena.h" 767#undef JEMALLOC_ARENA_INLINE_A 768#include "jemalloc/internal/tcache.h" 769#define JEMALLOC_ARENA_INLINE_B 770#include "jemalloc/internal/arena.h" 771#undef JEMALLOC_ARENA_INLINE_B 772#include "jemalloc/internal/hash.h" 773#include "jemalloc/internal/quarantine.h" 774 775#ifndef JEMALLOC_ENABLE_INLINE 776void *imallocx(size_t size, bool try_tcache, arena_t *arena); 777void *imalloc(size_t size); 778void *icallocx(size_t size, bool try_tcache, arena_t *arena); 779void *icalloc(size_t size); 780void *ipallocx(size_t usize, size_t alignment, bool zero, bool try_tcache, 781 arena_t *arena); 782void *ipalloc(size_t usize, size_t alignment, bool zero); 783size_t isalloc(const void *ptr, bool demote); 784size_t ivsalloc(const void *ptr, bool demote); 785size_t u2rz(size_t usize); 786size_t p2rz(const void *ptr); 787void idallocx(void *ptr, bool try_tcache); 788void idalloc(void *ptr); 789void iqallocx(void *ptr, bool try_tcache); 790void iqalloc(void *ptr); 791void *irallocx(void *ptr, size_t size, size_t extra, size_t alignment, 792 bool zero, bool no_move, bool try_tcache_alloc, bool try_tcache_dalloc, 793 arena_t *arena); 794void *iralloc(void *ptr, size_t size, size_t extra, size_t alignment, 795 bool zero, bool no_move); 796malloc_tsd_protos(JEMALLOC_ATTR(unused), thread_allocated, thread_allocated_t) 797#endif 798 799#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) 800JEMALLOC_ALWAYS_INLINE void * 801imallocx(size_t size, bool try_tcache, arena_t *arena) 802{ 803 804 assert(size != 0); 805 806 if (size <= arena_maxclass) 807 return (arena_malloc(arena, size, false, try_tcache)); 808 else 809 return (huge_malloc(size, false)); 810} 811 812JEMALLOC_ALWAYS_INLINE void * 813imalloc(size_t size) 814{ 815 816 return (imallocx(size, true, NULL)); 817} 818 819JEMALLOC_ALWAYS_INLINE void * 820icallocx(size_t size, bool try_tcache, arena_t *arena) 821{ 822 823 if (size <= arena_maxclass) 824 return (arena_malloc(arena, size, true, try_tcache)); 825 else 826 return (huge_malloc(size, true)); 827} 828 829JEMALLOC_ALWAYS_INLINE void * 830icalloc(size_t size) 831{ 832 833 return (icallocx(size, true, NULL)); 834} 835 836JEMALLOC_ALWAYS_INLINE void * 837ipallocx(size_t usize, size_t alignment, bool zero, bool try_tcache, 838 arena_t *arena) 839{ 840 void *ret; 841 842 assert(usize != 0); 843 assert(usize == sa2u(usize, alignment)); 844 845 if (usize <= arena_maxclass && alignment <= PAGE) 846 ret = arena_malloc(arena, usize, zero, try_tcache); 847 else { 848 if (usize <= arena_maxclass) { 849 ret = arena_palloc(choose_arena(arena), usize, 850 alignment, zero); 851 } else if (alignment <= chunksize) 852 ret = huge_malloc(usize, zero); 853 else 854 ret = huge_palloc(usize, alignment, zero); 855 } 856 857 assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret); 858 return (ret); 859} 860 861JEMALLOC_ALWAYS_INLINE void * 862ipalloc(size_t usize, size_t alignment, bool zero) 863{ 864 865 return (ipallocx(usize, alignment, zero, true, NULL)); 866} 867 868/* 869 * Typical usage: 870 * void *ptr = [...] 871 * size_t sz = isalloc(ptr, config_prof); 872 */ 873JEMALLOC_ALWAYS_INLINE size_t 874isalloc(const void *ptr, bool demote) 875{ 876 size_t ret; 877 arena_chunk_t *chunk; 878 879 assert(ptr != NULL); 880 /* Demotion only makes sense if config_prof is true. */ 881 assert(config_prof || demote == false); 882 883 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 884 if (chunk != ptr) 885 ret = arena_salloc(ptr, demote); 886 else 887 ret = huge_salloc(ptr); 888 889 return (ret); 890} 891 892JEMALLOC_ALWAYS_INLINE size_t 893ivsalloc(const void *ptr, bool demote) 894{ 895 896 /* Return 0 if ptr is not within a chunk managed by jemalloc. */ 897 if (rtree_get(chunks_rtree, (uintptr_t)CHUNK_ADDR2BASE(ptr)) == NULL) 898 return (0); 899 900 return (isalloc(ptr, demote)); 901} 902 903JEMALLOC_INLINE size_t 904u2rz(size_t usize) 905{ 906 size_t ret; 907 908 if (usize <= SMALL_MAXCLASS) { 909 size_t binind = SMALL_SIZE2BIN(usize); 910 ret = arena_bin_info[binind].redzone_size; 911 } else 912 ret = 0; 913 914 return (ret); 915} 916 917JEMALLOC_INLINE size_t 918p2rz(const void *ptr) 919{ 920 size_t usize = isalloc(ptr, false); 921 922 return (u2rz(usize)); 923} 924 925JEMALLOC_ALWAYS_INLINE void 926idallocx(void *ptr, bool try_tcache) 927{ 928 arena_chunk_t *chunk; 929 930 assert(ptr != NULL); 931 932 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 933 if (chunk != ptr) 934 arena_dalloc(chunk->arena, chunk, ptr, try_tcache); 935 else 936 huge_dalloc(ptr, true); 937} 938 939JEMALLOC_ALWAYS_INLINE void 940idalloc(void *ptr) 941{ 942 943 idallocx(ptr, true); 944} 945 946JEMALLOC_ALWAYS_INLINE void 947iqallocx(void *ptr, bool try_tcache) 948{ 949 950 if (config_fill && opt_quarantine) 951 quarantine(ptr); 952 else 953 idallocx(ptr, try_tcache); 954} 955 956JEMALLOC_ALWAYS_INLINE void 957iqalloc(void *ptr) 958{ 959 960 iqallocx(ptr, true); 961} 962 963JEMALLOC_ALWAYS_INLINE void * 964irallocx(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, 965 bool no_move, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena) 966{ 967 void *ret; 968 size_t oldsize; 969 970 assert(ptr != NULL); 971 assert(size != 0); 972 973 oldsize = isalloc(ptr, config_prof); 974 975 if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) 976 != 0) { 977 size_t usize, copysize; 978 979 /* 980 * Existing object alignment is inadequate; allocate new space 981 * and copy. 982 */ 983 if (no_move) 984 return (NULL); 985 usize = sa2u(size + extra, alignment); 986 if (usize == 0) 987 return (NULL); 988 ret = ipallocx(usize, alignment, zero, try_tcache_alloc, arena); 989 if (ret == NULL) { 990 if (extra == 0) 991 return (NULL); 992 /* Try again, without extra this time. */ 993 usize = sa2u(size, alignment); 994 if (usize == 0) 995 return (NULL); 996 ret = ipallocx(usize, alignment, zero, try_tcache_alloc, 997 arena); 998 if (ret == NULL) 999 return (NULL); 1000 } 1001 /* 1002 * Copy at most size bytes (not size+extra), since the caller 1003 * has no expectation that the extra bytes will be reliably 1004 * preserved. 1005 */ 1006 copysize = (size < oldsize) ? size : oldsize; 1007 memcpy(ret, ptr, copysize); 1008 iqallocx(ptr, try_tcache_dalloc); 1009 return (ret); 1010 } 1011 1012 if (no_move) { 1013 if (size <= arena_maxclass) { 1014 return (arena_ralloc_no_move(ptr, oldsize, size, 1015 extra, zero)); 1016 } else { 1017 return (huge_ralloc_no_move(ptr, oldsize, size, 1018 extra)); 1019 } 1020 } else { 1021 if (size + extra <= arena_maxclass) { 1022 return (arena_ralloc(arena, ptr, oldsize, size, extra, 1023 alignment, zero, try_tcache_alloc, 1024 try_tcache_dalloc)); 1025 } else { 1026 return (huge_ralloc(ptr, oldsize, size, extra, 1027 alignment, zero, try_tcache_dalloc)); 1028 } 1029 } 1030} 1031 1032JEMALLOC_ALWAYS_INLINE void * 1033iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, 1034 bool no_move) 1035{ 1036 1037 return (irallocx(ptr, size, extra, alignment, zero, no_move, true, true, 1038 NULL)); 1039} 1040 1041malloc_tsd_externs(thread_allocated, thread_allocated_t) 1042malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, thread_allocated, thread_allocated_t, 1043 THREAD_ALLOCATED_INITIALIZER, malloc_tsd_no_cleanup) 1044#endif 1045 1046#include "jemalloc/internal/prof.h" 1047 1048#undef JEMALLOC_H_INLINES 1049/******************************************************************************/ 1050#endif /* JEMALLOC_INTERNAL_H */ 1051