15daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#ifndef JEMALLOC_INTERNAL_H 25daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define JEMALLOC_INTERNAL_H 35daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 45daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc_internal_defs.h" 59ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#include "jemalloc/internal/jemalloc_internal_decls.h" 65daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 75daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#ifdef JEMALLOC_UTRACE 85daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include <sys/ktrace.h> 95daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 105daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 115daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define JEMALLOC_NO_DEMANGLE 125daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#ifdef JEMALLOC_JET 135daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# define JEMALLOC_N(n) jet_##n 145daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# include "jemalloc/internal/public_namespace.h" 155daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# define JEMALLOC_NO_RENAME 165daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# include "../jemalloc.h" 175daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# undef JEMALLOC_NO_RENAME 185daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#else 195daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# define JEMALLOC_N(n) je_##n 205daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# include "../jemalloc.h" 215daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 225daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/private_namespace.h" 235daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 245daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrisstatic const bool config_debug = 255daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#ifdef JEMALLOC_DEBUG 265daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris true 275daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#else 285daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris false 295daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 305daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris ; 315daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrisstatic const bool have_dss = 325daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#ifdef JEMALLOC_DSS 335daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris true 345daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#else 355daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris false 365daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 375daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris ; 385daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrisstatic const bool config_fill = 395daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#ifdef JEMALLOC_FILL 405daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris true 415daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#else 425daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris false 435daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 445daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris ; 455daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrisstatic const bool config_lazy_lock = 465daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#ifdef JEMALLOC_LAZY_LOCK 475daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris true 485daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#else 495daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris false 505daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 515daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris ; 52473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferrisstatic const char * const config_malloc_conf = JEMALLOC_CONFIG_MALLOC_CONF; 535daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrisstatic const bool config_prof = 545daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#ifdef JEMALLOC_PROF 555daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris true 565daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#else 575daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris false 585daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 595daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris ; 605daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrisstatic const bool config_prof_libgcc = 615daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#ifdef JEMALLOC_PROF_LIBGCC 625daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris true 635daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#else 645daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris false 655daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 665daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris ; 675daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrisstatic const bool config_prof_libunwind = 685daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#ifdef JEMALLOC_PROF_LIBUNWIND 695daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris true 705daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#else 715daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris false 725daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 735daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris ; 746f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferrisstatic const bool maps_coalesce = 756f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris#ifdef JEMALLOC_MAPS_COALESCE 766f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris true 776f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris#else 786f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris false 796f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris#endif 806f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris ; 815daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrisstatic const bool config_munmap = 825daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#ifdef JEMALLOC_MUNMAP 835daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris true 845daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#else 855daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris false 865daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 875daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris ; 885daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrisstatic const bool config_stats = 895daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#ifdef JEMALLOC_STATS 905daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris true 915daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#else 925daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris false 935daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 945daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris ; 955daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrisstatic const bool config_tcache = 965daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#ifdef JEMALLOC_TCACHE 975daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris true 985daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#else 995daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris false 1005daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 1015daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris ; 1025daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrisstatic const bool config_tls = 1035daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#ifdef JEMALLOC_TLS 1045daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris true 1055daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#else 1065daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris false 1075daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 1085daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris ; 1095daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrisstatic const bool config_utrace = 1105daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#ifdef JEMALLOC_UTRACE 1115daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris true 1125daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#else 1135daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris false 1145daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 1155daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris ; 1165daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrisstatic const bool config_valgrind = 1175daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#ifdef JEMALLOC_VALGRIND 1185daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris true 1195daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#else 1205daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris false 1215daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 1225daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris ; 1235daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrisstatic const bool config_xmalloc = 1245daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#ifdef JEMALLOC_XMALLOC 1255daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris true 1265daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#else 1275daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris false 1285daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 1295daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris ; 1305daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrisstatic const bool config_ivsalloc = 1315daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#ifdef JEMALLOC_IVSALLOC 1325daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris true 1335daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#else 1345daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris false 1355daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 1365daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris ; 137cb4217087d4d5b1271f8316515f5e71f087b3162Christopher Ferrisstatic const bool config_cache_oblivious = 138cb4217087d4d5b1271f8316515f5e71f087b3162Christopher Ferris#ifdef JEMALLOC_CACHE_OBLIVIOUS 139cb4217087d4d5b1271f8316515f5e71f087b3162Christopher Ferris true 140cb4217087d4d5b1271f8316515f5e71f087b3162Christopher Ferris#else 141cb4217087d4d5b1271f8316515f5e71f087b3162Christopher Ferris false 142cb4217087d4d5b1271f8316515f5e71f087b3162Christopher Ferris#endif 143cb4217087d4d5b1271f8316515f5e71f087b3162Christopher Ferris ; 1445daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 1459eda39002ea8303855150075b8e1c5495c0aaf38Chih-Hung Hsieh#ifdef JEMALLOC_C11ATOMICS 1469eda39002ea8303855150075b8e1c5495c0aaf38Chih-Hung Hsieh#include <stdatomic.h> 1479eda39002ea8303855150075b8e1c5495c0aaf38Chih-Hung Hsieh#endif 1489eda39002ea8303855150075b8e1c5495c0aaf38Chih-Hung Hsieh 1495daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#ifdef JEMALLOC_ATOMIC9 1505daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include <machine/atomic.h> 1515daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 1525daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 1535daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN)) 1545daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include <libkern/OSAtomic.h> 1555daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 1565daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 1575daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#ifdef JEMALLOC_ZONE 1585daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include <mach/mach_error.h> 1595daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include <mach/mach_init.h> 1605daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include <mach/vm_map.h> 1615daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include <malloc/malloc.h> 1625daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 1635daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 164354524779686d6d5518078fd681110157631c941Christopher Ferris#include "jemalloc/internal/ph.h" 165fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris#ifndef __PGI 1665daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define RB_COMPACT 167fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris#endif 1685daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/rb.h" 1695daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/qr.h" 1705daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/ql.h" 1715daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 1725daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris/* 1735daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * jemalloc can conceptually be broken into components (arena, tcache, etc.), 1745daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * but there are circular dependencies that cannot be broken without 1755daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * substantial performance degradation. In order to reduce the effect on 1765daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * visual code flow, read the header files in multiple passes, with one of the 1775daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * following cpp variables defined during each pass: 1785daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * 1795daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * JEMALLOC_H_TYPES : Preprocessor-defined constants and psuedo-opaque data 1805daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * types. 1815daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * JEMALLOC_H_STRUCTS : Data structures. 1825daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * JEMALLOC_H_EXTERNS : Extern data declarations and function prototypes. 1835daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * JEMALLOC_H_INLINES : Inline functions. 1845daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris */ 1855daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris/******************************************************************************/ 1865daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define JEMALLOC_H_TYPES 1875daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 1885daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/jemalloc_internal_macros.h" 1895daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 190fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris/* Page size index type. */ 191fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferristypedef unsigned pszind_t; 192fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris 19396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris/* Size class index type. */ 1946f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferristypedef unsigned szind_t; 19596d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 19696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris/* 19796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * Flags bits: 19896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * 19996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * a: arena 20096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * t: tcache 20196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * 0: unused 20296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * z: zero 20396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * n: alignment 20496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * 20596d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * aaaaaaaa aaaatttt tttttttt 0znnnnnn 20696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris */ 20796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#define MALLOCX_ARENA_MASK ((int)~0xfffff) 20896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#define MALLOCX_ARENA_MAX 0xffe 20996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#define MALLOCX_TCACHE_MASK ((int)~0xfff000ffU) 21096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#define MALLOCX_TCACHE_MAX 0xffd 2115daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define MALLOCX_LG_ALIGN_MASK ((int)0x3f) 21296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris/* Use MALLOCX_ALIGN_GET() if alignment may not be specified in flags. */ 21396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#define MALLOCX_ALIGN_GET_SPECIFIED(flags) \ 21496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK)) 21596d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#define MALLOCX_ALIGN_GET(flags) \ 21696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris (MALLOCX_ALIGN_GET_SPECIFIED(flags) & (SIZE_T_MAX-1)) 21796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#define MALLOCX_ZERO_GET(flags) \ 21896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris ((bool)(flags & MALLOCX_ZERO)) 21996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 22096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#define MALLOCX_TCACHE_GET(flags) \ 22196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris (((unsigned)((flags & MALLOCX_TCACHE_MASK) >> 8)) - 2) 22296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#define MALLOCX_ARENA_GET(flags) \ 22396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris (((unsigned)(((unsigned)flags) >> 20)) - 1) 2245daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 2255daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris/* Smallest size class to support. */ 2265daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define TINY_MIN (1U << LG_TINY_MIN) 2275daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 2285daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris/* 22996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * Minimum allocation alignment is 2^LG_QUANTUM bytes (ignoring tiny size 2305daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * classes). 2315daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris */ 2325daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#ifndef LG_QUANTUM 2335daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# if (defined(__i386__) || defined(_M_IX86)) 2345daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# define LG_QUANTUM 4 2355daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# endif 2365daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# ifdef __ia64__ 2375daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# define LG_QUANTUM 4 2385daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# endif 2395daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# ifdef __alpha__ 2405daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# define LG_QUANTUM 4 2415daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# endif 242fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris# if (defined(__sparc64__) || defined(__sparcv9) || defined(__sparc_v9__)) 2435daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# define LG_QUANTUM 4 2445daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# endif 2455daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64)) 2465daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# define LG_QUANTUM 4 2475daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# endif 2485daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# ifdef __arm__ 2495daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# define LG_QUANTUM 3 2505daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# endif 2515daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# ifdef __aarch64__ 2525daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# define LG_QUANTUM 4 2535daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# endif 2545daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# ifdef __hppa__ 2555daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# define LG_QUANTUM 4 2565daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# endif 2575daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# ifdef __mips__ 2585daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# define LG_QUANTUM 3 2595daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# endif 26096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris# ifdef __or1k__ 26196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris# define LG_QUANTUM 3 26296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris# endif 2635daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# ifdef __powerpc__ 2645daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# define LG_QUANTUM 4 2655daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# endif 266354524779686d6d5518078fd681110157631c941Christopher Ferris# ifdef __riscv__ 267354524779686d6d5518078fd681110157631c941Christopher Ferris# define LG_QUANTUM 4 268354524779686d6d5518078fd681110157631c941Christopher Ferris# endif 2695daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# ifdef __s390__ 2705daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# define LG_QUANTUM 4 2715daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# endif 2725daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# ifdef __SH4__ 2735daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# define LG_QUANTUM 4 2745daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# endif 2755daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# ifdef __tile__ 2765daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# define LG_QUANTUM 4 2775daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# endif 2789ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris# ifdef __le32__ 2799ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris# define LG_QUANTUM 4 2809ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris# endif 2815daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# ifndef LG_QUANTUM 28296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris# error "Unknown minimum alignment for architecture; specify via " 28396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris "--with-lg-quantum" 2845daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# endif 2855daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 2865daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 2875daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define QUANTUM ((size_t)(1U << LG_QUANTUM)) 2885daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define QUANTUM_MASK (QUANTUM - 1) 2895daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 2905daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris/* Return the smallest quantum multiple that is >= a. */ 2915daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define QUANTUM_CEILING(a) \ 2925daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris (((a) + QUANTUM_MASK) & ~QUANTUM_MASK) 2935daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 2945daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define LONG ((size_t)(1U << LG_SIZEOF_LONG)) 2955daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define LONG_MASK (LONG - 1) 2965daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 2975daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris/* Return the smallest long multiple that is >= a. */ 2985daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define LONG_CEILING(a) \ 2995daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris (((a) + LONG_MASK) & ~LONG_MASK) 3005daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 3015daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define SIZEOF_PTR (1U << LG_SIZEOF_PTR) 3025daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define PTR_MASK (SIZEOF_PTR - 1) 3035daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 3045daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris/* Return the smallest (void *) multiple that is >= a. */ 3055daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define PTR_CEILING(a) \ 3065daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris (((a) + PTR_MASK) & ~PTR_MASK) 3075daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 3085daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris/* 3095daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * Maximum size of L1 cache line. This is used to avoid cache line aliasing. 3105daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * In addition, this controls the spacing of cacheline-spaced size classes. 3115daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * 3125daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * CACHELINE cannot be based on LG_CACHELINE because __declspec(align()) can 3135daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * only handle raw constants. 3145daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris */ 3155daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define LG_CACHELINE 6 3165daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define CACHELINE 64 3175daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define CACHELINE_MASK (CACHELINE - 1) 3185daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 3195daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris/* Return the smallest cacheline multiple that is >= s. */ 3205daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define CACHELINE_CEILING(s) \ 3215daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris (((s) + CACHELINE_MASK) & ~CACHELINE_MASK) 3225daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 32396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris/* Page size. LG_PAGE is determined by the configure script. */ 3245daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#ifdef PAGE_MASK 3255daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# undef PAGE_MASK 3265daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 32796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#define PAGE ((size_t)(1U << LG_PAGE)) 3285daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define PAGE_MASK ((size_t)(PAGE - 1)) 3295daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 330ad6417c653ea7eeaebe6a0d56a31aa1cf1c93336Christopher Ferris/* Return the page base address for the page containing address a. */ 331ad6417c653ea7eeaebe6a0d56a31aa1cf1c93336Christopher Ferris#define PAGE_ADDR2BASE(a) \ 332ad6417c653ea7eeaebe6a0d56a31aa1cf1c93336Christopher Ferris ((void *)((uintptr_t)(a) & ~PAGE_MASK)) 333ad6417c653ea7eeaebe6a0d56a31aa1cf1c93336Christopher Ferris 3345daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris/* Return the smallest pagesize multiple that is >= s. */ 3355daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define PAGE_CEILING(s) \ 3365daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris (((s) + PAGE_MASK) & ~PAGE_MASK) 3375daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 3385daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris/* Return the nearest aligned address at or below a. */ 3395daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define ALIGNMENT_ADDR2BASE(a, alignment) \ 340bf9b0180f3e75b710d1f419fff343885ee1b3880Christopher Ferris ((void *)((uintptr_t)(a) & ((~(alignment)) + 1))) 3415daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 3425daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris/* Return the offset between a and the nearest aligned address at or below a. */ 3435daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define ALIGNMENT_ADDR2OFFSET(a, alignment) \ 3445daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris ((size_t)((uintptr_t)(a) & (alignment - 1))) 3455daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 3465daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris/* Return the smallest alignment multiple that is >= s. */ 3475daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define ALIGNMENT_CEILING(s, alignment) \ 348bf9b0180f3e75b710d1f419fff343885ee1b3880Christopher Ferris (((s) + (alignment - 1)) & ((~(alignment)) + 1)) 3495daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 35096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris/* Declare a variable-length array. */ 3515daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#if __STDC_VERSION__ < 199901L 3525daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# ifdef _MSC_VER 3535daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# include <malloc.h> 3545daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# define alloca _alloca 3555daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# else 3565daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# ifdef JEMALLOC_HAS_ALLOCA_H 3575daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# include <alloca.h> 3585daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# else 3595daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# include <stdlib.h> 3605daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# endif 3615daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# endif 3625daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris# define VARIABLE_ARRAY(type, name, count) \ 3639ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris type *name = alloca(sizeof(type) * (count)) 3645daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#else 3659ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris# define VARIABLE_ARRAY(type, name, count) type name[(count)] 3665daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 3675daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 368473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris#include "jemalloc/internal/nstime.h" 3695daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/valgrind.h" 3705daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/util.h" 3715daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/atomic.h" 372fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris#include "jemalloc/internal/spin.h" 3735daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/prng.h" 374473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris#include "jemalloc/internal/ticker.h" 3755daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/ckh.h" 3765daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/size_classes.h" 377473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris#include "jemalloc/internal/smoothstep.h" 3785daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/stats.h" 3795daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/ctl.h" 380354524779686d6d5518078fd681110157631c941Christopher Ferris#include "jemalloc/internal/witness.h" 3815daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/mutex.h" 3825daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/tsd.h" 3835daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/mb.h" 3845daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/extent.h" 3855daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/arena.h" 3865daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/bitmap.h" 3875daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/base.h" 38896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#include "jemalloc/internal/rtree.h" 3896f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris#include "jemalloc/internal/pages.h" 3905daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/chunk.h" 3915daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/huge.h" 3925daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/tcache.h" 3935daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/hash.h" 3945daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/quarantine.h" 3955daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/prof.h" 3965daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 3975daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#undef JEMALLOC_H_TYPES 3985daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris/******************************************************************************/ 3995daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define JEMALLOC_H_STRUCTS 4005daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 401473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris#include "jemalloc/internal/nstime.h" 4025daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/valgrind.h" 4035daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/util.h" 4045daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/atomic.h" 405fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris#include "jemalloc/internal/spin.h" 4065daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/prng.h" 407473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris#include "jemalloc/internal/ticker.h" 4085daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/ckh.h" 4095daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/size_classes.h" 410473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris#include "jemalloc/internal/smoothstep.h" 4115daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/stats.h" 4125daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/ctl.h" 413354524779686d6d5518078fd681110157631c941Christopher Ferris#include "jemalloc/internal/witness.h" 4145daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/mutex.h" 4155daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/mb.h" 4165daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/bitmap.h" 41796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#define JEMALLOC_ARENA_STRUCTS_A 41896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#include "jemalloc/internal/arena.h" 41996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#undef JEMALLOC_ARENA_STRUCTS_A 4205daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/extent.h" 42196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#define JEMALLOC_ARENA_STRUCTS_B 4225daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/arena.h" 42396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#undef JEMALLOC_ARENA_STRUCTS_B 4245daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/base.h" 42596d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#include "jemalloc/internal/rtree.h" 4266f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris#include "jemalloc/internal/pages.h" 4275daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/chunk.h" 4285daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/huge.h" 4295daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/tcache.h" 4305daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/hash.h" 4315daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/quarantine.h" 4325daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/prof.h" 4335daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 43496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#include "jemalloc/internal/tsd.h" 4355daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 4365daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#undef JEMALLOC_H_STRUCTS 4375daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris/******************************************************************************/ 4385daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define JEMALLOC_H_EXTERNS 4395daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 4405daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrisextern bool opt_abort; 44196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisextern const char *opt_junk; 44296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisextern bool opt_junk_alloc; 44396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisextern bool opt_junk_free; 4445daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrisextern size_t opt_quarantine; 4455daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrisextern bool opt_redzone; 4465daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrisextern bool opt_utrace; 4475daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrisextern bool opt_xmalloc; 4485daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrisextern bool opt_zero; 449473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferrisextern unsigned opt_narenas; 4505daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 4515daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrisextern bool in_valgrind; 4525daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 4535daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris/* Number of CPUs. */ 454473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferrisextern unsigned ncpus; 455473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris 456354524779686d6d5518078fd681110157631c941Christopher Ferris/* Number of arenas used for automatic multiplexing of threads and arenas. */ 457354524779686d6d5518078fd681110157631c941Christopher Ferrisextern unsigned narenas_auto; 458354524779686d6d5518078fd681110157631c941Christopher Ferris 459473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris/* 460473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris * Arenas that are used to service external requests. Not all elements of the 461473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris * arenas array are necessarily used; arenas are created lazily as needed. 462473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris */ 463473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferrisextern arena_t **arenas; 4645daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 4655daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris/* 466fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris * pind2sz_tab encodes the same information as could be computed by 467fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris * pind2sz_compute(). 468fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris */ 469fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferrisextern size_t const pind2sz_tab[NPSIZES]; 470fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris/* 47196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * index2size_tab encodes the same information as could be computed (at 47296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * unacceptable cost in some code paths) by index2size_compute(). 4730b9d74bfed176f3c83114086a9dbf207cdeeb1f1Christopher Ferris */ 474fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferrisextern size_t const index2size_tab[NSIZES]; 47596d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris/* 47696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * size2index_tab is a compact lookup table that rounds request sizes up to 47796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * size classes. In order to reduce cache footprint, the table is compressed, 47896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * and all accesses are via size2index(). 47996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris */ 48096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisextern uint8_t const size2index_tab[]; 48196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 482fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferrisarena_t *a0get(void); 48396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisvoid *a0malloc(size_t size); 48496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisvoid a0dalloc(void *ptr); 48596d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisvoid *bootstrap_malloc(size_t size); 48696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisvoid *bootstrap_calloc(size_t num, size_t size); 48796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisvoid bootstrap_free(void *ptr); 48896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisunsigned narenas_total_get(void); 489354524779686d6d5518078fd681110157631c941Christopher Ferrisarena_t *arena_init(tsdn_t *tsdn, unsigned ind); 490473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferrisarena_tdata_t *arena_tdata_get_hard(tsd_t *tsd, unsigned ind); 491354524779686d6d5518078fd681110157631c941Christopher Ferrisarena_t *arena_choose_hard(tsd_t *tsd, bool internal); 49296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisvoid arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind); 49396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisvoid thread_allocated_cleanup(tsd_t *tsd); 49496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisvoid thread_deallocated_cleanup(tsd_t *tsd); 495354524779686d6d5518078fd681110157631c941Christopher Ferrisvoid iarena_cleanup(tsd_t *tsd); 49696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisvoid arena_cleanup(tsd_t *tsd); 497473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferrisvoid arenas_tdata_cleanup(tsd_t *tsd); 498473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferrisvoid narenas_tdata_cleanup(tsd_t *tsd); 499473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferrisvoid arenas_tdata_bypass_cleanup(tsd_t *tsd); 5005daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrisvoid jemalloc_prefork(void); 5015daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrisvoid jemalloc_postfork_parent(void); 5025daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrisvoid jemalloc_postfork_child(void); 5035daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 504473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris#include "jemalloc/internal/nstime.h" 5055daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/valgrind.h" 5065daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/util.h" 5075daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/atomic.h" 508fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris#include "jemalloc/internal/spin.h" 5095daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/prng.h" 510473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris#include "jemalloc/internal/ticker.h" 5115daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/ckh.h" 5125daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/size_classes.h" 513473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris#include "jemalloc/internal/smoothstep.h" 5145daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/stats.h" 5155daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/ctl.h" 516354524779686d6d5518078fd681110157631c941Christopher Ferris#include "jemalloc/internal/witness.h" 5175daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/mutex.h" 5185daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/mb.h" 5195daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/bitmap.h" 5205daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/extent.h" 5215daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/arena.h" 5225daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/base.h" 52396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#include "jemalloc/internal/rtree.h" 5246f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris#include "jemalloc/internal/pages.h" 5255daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/chunk.h" 5265daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/huge.h" 5275daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/tcache.h" 5285daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/hash.h" 5295daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/quarantine.h" 5305daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/prof.h" 53196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#include "jemalloc/internal/tsd.h" 5325daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 5335daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#undef JEMALLOC_H_EXTERNS 5345daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris/******************************************************************************/ 5355daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#define JEMALLOC_H_INLINES 5365daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 537473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris#include "jemalloc/internal/nstime.h" 5385daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/valgrind.h" 5395daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/util.h" 5405daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/atomic.h" 541fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris#include "jemalloc/internal/spin.h" 5425daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/prng.h" 543473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris#include "jemalloc/internal/ticker.h" 5445daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/ckh.h" 5455daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/size_classes.h" 546473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris#include "jemalloc/internal/smoothstep.h" 5475daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/stats.h" 5485daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/ctl.h" 5495daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/tsd.h" 550354524779686d6d5518078fd681110157631c941Christopher Ferris#include "jemalloc/internal/witness.h" 551354524779686d6d5518078fd681110157631c941Christopher Ferris#include "jemalloc/internal/mutex.h" 5525daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/mb.h" 5535daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/extent.h" 5545daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/base.h" 55596d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#include "jemalloc/internal/rtree.h" 5566f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris#include "jemalloc/internal/pages.h" 5575daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/chunk.h" 5585daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/huge.h" 5595daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 5605daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#ifndef JEMALLOC_ENABLE_INLINE 561fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferrispszind_t psz2ind(size_t psz); 562fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferrissize_t pind2sz_compute(pszind_t pind); 563fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferrissize_t pind2sz_lookup(pszind_t pind); 564fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferrissize_t pind2sz(pszind_t pind); 565fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferrissize_t psz2u(size_t psz); 5666f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferrisszind_t size2index_compute(size_t size); 5676f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferrisszind_t size2index_lookup(size_t size); 5686f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferrisszind_t size2index(size_t size); 5696f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferrissize_t index2size_compute(szind_t index); 5706f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferrissize_t index2size_lookup(szind_t index); 5716f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferrissize_t index2size(szind_t index); 57296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrissize_t s2u_compute(size_t size); 57396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrissize_t s2u_lookup(size_t size); 5745daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrissize_t s2u(size_t size); 5755daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrissize_t sa2u(size_t size, size_t alignment); 576354524779686d6d5518078fd681110157631c941Christopher Ferrisarena_t *arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal); 57796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisarena_t *arena_choose(tsd_t *tsd, arena_t *arena); 578fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferrisarena_t *arena_ichoose(tsd_t *tsd, arena_t *arena); 579473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferrisarena_tdata_t *arena_tdata_get(tsd_t *tsd, unsigned ind, 58096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris bool refresh_if_missing); 581354524779686d6d5518078fd681110157631c941Christopher Ferrisarena_t *arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing); 582473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferristicker_t *decay_ticker_get(tsd_t *tsd, unsigned ind); 5835daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 5845daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 5855daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) 586fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher FerrisJEMALLOC_INLINE pszind_t 587fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferrispsz2ind(size_t psz) 588fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris{ 589fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris 590fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris if (unlikely(psz > HUGE_MAXCLASS)) 591fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris return (NPSIZES); 592fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris { 593fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris pszind_t x = lg_floor((psz<<1)-1); 594fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris pszind_t shift = (x < LG_SIZE_CLASS_GROUP + LG_PAGE) ? 0 : x - 595fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris (LG_SIZE_CLASS_GROUP + LG_PAGE); 596fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris pszind_t grp = shift << LG_SIZE_CLASS_GROUP; 597fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris 598fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris pszind_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_PAGE + 1) ? 599fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris LG_PAGE : x - LG_SIZE_CLASS_GROUP - 1; 600fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris 601fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris size_t delta_inverse_mask = ZI(-1) << lg_delta; 602fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris pszind_t mod = ((((psz-1) & delta_inverse_mask) >> lg_delta)) & 603fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); 604fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris 605fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris pszind_t ind = grp + mod; 606fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris return (ind); 607fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris } 608fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris} 609fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris 610fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher FerrisJEMALLOC_INLINE size_t 611fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferrispind2sz_compute(pszind_t pind) 612fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris{ 613fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris 614fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris { 615fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris size_t grp = pind >> LG_SIZE_CLASS_GROUP; 616fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris size_t mod = pind & ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); 617fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris 618fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris size_t grp_size_mask = ~((!!grp)-1); 619fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris size_t grp_size = ((ZU(1) << (LG_PAGE + 620fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris (LG_SIZE_CLASS_GROUP-1))) << grp) & grp_size_mask; 621fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris 622fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris size_t shift = (grp == 0) ? 1 : grp; 623fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris size_t lg_delta = shift + (LG_PAGE-1); 624fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris size_t mod_size = (mod+1) << lg_delta; 625fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris 626fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris size_t sz = grp_size + mod_size; 627fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris return (sz); 628fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris } 629fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris} 630fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris 631fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher FerrisJEMALLOC_INLINE size_t 632fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferrispind2sz_lookup(pszind_t pind) 633fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris{ 634fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris size_t ret = (size_t)pind2sz_tab[pind]; 635fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris assert(ret == pind2sz_compute(pind)); 636fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris return (ret); 637fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris} 638fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris 639fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher FerrisJEMALLOC_INLINE size_t 640fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferrispind2sz(pszind_t pind) 641fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris{ 642fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris 643fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris assert(pind < NPSIZES); 644fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris return (pind2sz_lookup(pind)); 645fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris} 646fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris 647fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher FerrisJEMALLOC_INLINE size_t 648fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferrispsz2u(size_t psz) 649fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris{ 650fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris 651fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris if (unlikely(psz > HUGE_MAXCLASS)) 652fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris return (0); 653fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris { 654fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris size_t x = lg_floor((psz<<1)-1); 655fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_PAGE + 1) ? 656fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris LG_PAGE : x - LG_SIZE_CLASS_GROUP - 1; 657fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris size_t delta = ZU(1) << lg_delta; 658fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris size_t delta_mask = delta - 1; 659fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris size_t usize = (psz + delta_mask) & ~delta_mask; 660fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris return (usize); 661fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris } 662fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris} 663fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris 6646f50cbc975f782b7576ed699a7c2b123655d1b95Christopher FerrisJEMALLOC_INLINE szind_t 66596d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrissize2index_compute(size_t size) 66696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris{ 66796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 668fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris if (unlikely(size > HUGE_MAXCLASS)) 669fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris return (NSIZES); 67096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#if (NTBINS != 0) 67196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { 672473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris szind_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; 673473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris szind_t lg_ceil = lg_floor(pow2_ceil_zu(size)); 67496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris return (lg_ceil < lg_tmin ? 0 : lg_ceil - lg_tmin); 6756f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris } 67696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#endif 67796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris { 678fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris szind_t x = lg_floor((size<<1)-1); 679473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris szind_t shift = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM) ? 0 : 68096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris x - (LG_SIZE_CLASS_GROUP + LG_QUANTUM); 681473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris szind_t grp = shift << LG_SIZE_CLASS_GROUP; 68296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 683473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris szind_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1) 68496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1; 68596d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 68696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris size_t delta_inverse_mask = ZI(-1) << lg_delta; 687473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris szind_t mod = ((((size-1) & delta_inverse_mask) >> lg_delta)) & 68896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); 68996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 690473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris szind_t index = NTBINS + grp + mod; 69196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris return (index); 69296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris } 69396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris} 69496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 6956f50cbc975f782b7576ed699a7c2b123655d1b95Christopher FerrisJEMALLOC_ALWAYS_INLINE szind_t 69696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrissize2index_lookup(size_t size) 69796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris{ 69896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 69996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris assert(size <= LOOKUP_MAXCLASS); 70096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris { 701473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris szind_t ret = (size2index_tab[(size-1) >> LG_TINY_MIN]); 70296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris assert(ret == size2index_compute(size)); 70396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris return (ret); 70496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris } 70596d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris} 70696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 7076f50cbc975f782b7576ed699a7c2b123655d1b95Christopher FerrisJEMALLOC_ALWAYS_INLINE szind_t 70896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrissize2index(size_t size) 70996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris{ 71096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 71196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris assert(size > 0); 71296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris if (likely(size <= LOOKUP_MAXCLASS)) 71396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris return (size2index_lookup(size)); 7146f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris return (size2index_compute(size)); 71596d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris} 71696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 71796d58c8575eeac2a4a026e4d3629388573335bfaChristopher FerrisJEMALLOC_INLINE size_t 7186f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferrisindex2size_compute(szind_t index) 71996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris{ 72096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 72196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#if (NTBINS > 0) 72296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris if (index < NTBINS) 72396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris return (ZU(1) << (LG_TINY_MAXCLASS - NTBINS + 1 + index)); 72496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#endif 72596d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris { 72696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris size_t reduced_index = index - NTBINS; 72796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris size_t grp = reduced_index >> LG_SIZE_CLASS_GROUP; 72896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris size_t mod = reduced_index & ((ZU(1) << LG_SIZE_CLASS_GROUP) - 72996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 1); 73096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 73196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris size_t grp_size_mask = ~((!!grp)-1); 73296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris size_t grp_size = ((ZU(1) << (LG_QUANTUM + 73396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris (LG_SIZE_CLASS_GROUP-1))) << grp) & grp_size_mask; 73496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 73596d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris size_t shift = (grp == 0) ? 1 : grp; 73696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris size_t lg_delta = shift + (LG_QUANTUM-1); 73796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris size_t mod_size = (mod+1) << lg_delta; 73896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 73996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris size_t usize = grp_size + mod_size; 74096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris return (usize); 74196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris } 74296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris} 74396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 74496d58c8575eeac2a4a026e4d3629388573335bfaChristopher FerrisJEMALLOC_ALWAYS_INLINE size_t 7456f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferrisindex2size_lookup(szind_t index) 74696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris{ 74796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris size_t ret = (size_t)index2size_tab[index]; 74896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris assert(ret == index2size_compute(index)); 74996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris return (ret); 75096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris} 75196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 75296d58c8575eeac2a4a026e4d3629388573335bfaChristopher FerrisJEMALLOC_ALWAYS_INLINE size_t 7536f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferrisindex2size(szind_t index) 75496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris{ 75596d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 756473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris assert(index < NSIZES); 75796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris return (index2size_lookup(index)); 75896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris} 75996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 76096d58c8575eeac2a4a026e4d3629388573335bfaChristopher FerrisJEMALLOC_ALWAYS_INLINE size_t 76196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferriss2u_compute(size_t size) 76296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris{ 76396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 764fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris if (unlikely(size > HUGE_MAXCLASS)) 765fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris return (0); 76696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#if (NTBINS > 0) 76796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { 76896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; 769473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris size_t lg_ceil = lg_floor(pow2_ceil_zu(size)); 77096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris return (lg_ceil < lg_tmin ? (ZU(1) << lg_tmin) : 77196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris (ZU(1) << lg_ceil)); 7726f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris } 77396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#endif 77496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris { 775fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris size_t x = lg_floor((size<<1)-1); 77696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1) 77796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1; 77896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris size_t delta = ZU(1) << lg_delta; 77996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris size_t delta_mask = delta - 1; 78096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris size_t usize = (size + delta_mask) & ~delta_mask; 78196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris return (usize); 78296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris } 78396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris} 78496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 78596d58c8575eeac2a4a026e4d3629388573335bfaChristopher FerrisJEMALLOC_ALWAYS_INLINE size_t 78696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferriss2u_lookup(size_t size) 78796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris{ 78896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris size_t ret = index2size_lookup(size2index_lookup(size)); 78996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 79096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris assert(ret == s2u_compute(size)); 79196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris return (ret); 79296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris} 7935daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 7945daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris/* 7955daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * Compute usable size that would result from allocating an object with the 7965daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * specified size. 7975daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris */ 7985daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher FerrisJEMALLOC_ALWAYS_INLINE size_t 7995daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferriss2u(size_t size) 8005daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris{ 8015daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 80296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris assert(size > 0); 80396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris if (likely(size <= LOOKUP_MAXCLASS)) 80496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris return (s2u_lookup(size)); 8056f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris return (s2u_compute(size)); 8065daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris} 8075daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 8085daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris/* 8095daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * Compute usable size that would result from allocating an object with the 8105daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * specified size and alignment. 8115daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris */ 8125daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher FerrisJEMALLOC_ALWAYS_INLINE size_t 8135daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrissa2u(size_t size, size_t alignment) 8145daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris{ 8155daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris size_t usize; 8165daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 8175daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris assert(alignment != 0 && ((alignment - 1) & alignment) == 0); 8185daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 81996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris /* Try for a small size class. */ 82096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris if (size <= SMALL_MAXCLASS && alignment < PAGE) { 82196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris /* 82296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * Round size up to the nearest multiple of alignment. 82396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * 82496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * This done, we can take advantage of the fact that for each 82596d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * small size class, every object is aligned at the smallest 82696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * power of two that is non-zero in the base two representation 82796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * of the size. For example: 82896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * 82996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * Size | Base 2 | Minimum alignment 83096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * -----+----------+------------------ 83196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * 96 | 1100000 | 32 83296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * 144 | 10100000 | 32 83396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * 192 | 11000000 | 64 83496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris */ 83596d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris usize = s2u(ALIGNMENT_CEILING(size, alignment)); 83696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris if (usize < LARGE_MINCLASS) 83796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris return (usize); 8385daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris } 8395daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 84096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris /* Try for a large size class. */ 841ad6417c653ea7eeaebe6a0d56a31aa1cf1c93336Christopher Ferris if (likely(size <= large_maxclass) && likely(alignment < chunksize)) { 8425daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris /* 8435daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * We can't achieve subpage alignment, so round up alignment 84496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * to the minimum that can actually be supported. 8455daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris */ 8465daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris alignment = PAGE_CEILING(alignment); 84796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 84896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris /* Make sure result is a large size class. */ 84996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris usize = (size <= LARGE_MINCLASS) ? LARGE_MINCLASS : s2u(size); 8505daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 8515daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris /* 8525daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * Calculate the size of the over-size run that arena_palloc() 8535daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * would need to allocate in order to guarantee the alignment. 8545daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris */ 8556f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris if (usize + large_pad + alignment - PAGE <= arena_maxrun) 85696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris return (usize); 8575daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris } 8585daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 859473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris /* Huge size class. Beware of overflow. */ 860473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris 861473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris if (unlikely(alignment > HUGE_MAXCLASS)) 862473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris return (0); 86396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 86496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris /* 86596d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * We can't achieve subchunk alignment, so round up alignment to the 86696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * minimum that can actually be supported. 86796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris */ 86896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris alignment = CHUNK_CEILING(alignment); 8695daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 87096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris /* Make sure result is a huge size class. */ 87196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris if (size <= chunksize) 87296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris usize = chunksize; 87396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris else { 87496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris usize = s2u(size); 87596d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris if (usize < size) { 87696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris /* size_t overflow. */ 87796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris return (0); 87896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris } 87996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris } 8805daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 88196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris /* 88296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * Calculate the multi-chunk mapping that huge_palloc() would need in 88396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * order to guarantee the alignment. 88496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris */ 88596d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris if (usize + alignment - PAGE < usize) { 88696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris /* size_t overflow. */ 88796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris return (0); 88896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris } 88996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris return (usize); 8905daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris} 8915daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 8925daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris/* Choose an arena based on a per-thread value. */ 8935daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher FerrisJEMALLOC_INLINE arena_t * 894354524779686d6d5518078fd681110157631c941Christopher Ferrisarena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) 8955daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris{ 8965daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris arena_t *ret; 8975daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 8985daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris if (arena != NULL) 8995daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris return (arena); 9005daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 901354524779686d6d5518078fd681110157631c941Christopher Ferris ret = internal ? tsd_iarena_get(tsd) : tsd_arena_get(tsd); 902354524779686d6d5518078fd681110157631c941Christopher Ferris if (unlikely(ret == NULL)) 903354524779686d6d5518078fd681110157631c941Christopher Ferris ret = arena_choose_hard(tsd, internal); 9045daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 9055daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris return (ret); 9065daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris} 90796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 908354524779686d6d5518078fd681110157631c941Christopher FerrisJEMALLOC_INLINE arena_t * 909354524779686d6d5518078fd681110157631c941Christopher Ferrisarena_choose(tsd_t *tsd, arena_t *arena) 910354524779686d6d5518078fd681110157631c941Christopher Ferris{ 911354524779686d6d5518078fd681110157631c941Christopher Ferris 912354524779686d6d5518078fd681110157631c941Christopher Ferris return (arena_choose_impl(tsd, arena, false)); 913354524779686d6d5518078fd681110157631c941Christopher Ferris} 914354524779686d6d5518078fd681110157631c941Christopher Ferris 915354524779686d6d5518078fd681110157631c941Christopher FerrisJEMALLOC_INLINE arena_t * 916fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferrisarena_ichoose(tsd_t *tsd, arena_t *arena) 917354524779686d6d5518078fd681110157631c941Christopher Ferris{ 918354524779686d6d5518078fd681110157631c941Christopher Ferris 919fb1f094f163a3bf15d8958ba845e83559c0e6dfeChristopher Ferris return (arena_choose_impl(tsd, arena, true)); 920354524779686d6d5518078fd681110157631c941Christopher Ferris} 921354524779686d6d5518078fd681110157631c941Christopher Ferris 922473a1853074261bb265060e04deeb8efb0fe6facChristopher FerrisJEMALLOC_INLINE arena_tdata_t * 923473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferrisarena_tdata_get(tsd_t *tsd, unsigned ind, bool refresh_if_missing) 92496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris{ 925473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris arena_tdata_t *tdata; 926473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris arena_tdata_t *arenas_tdata = tsd_arenas_tdata_get(tsd); 92796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 928473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris if (unlikely(arenas_tdata == NULL)) { 929473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris /* arenas_tdata hasn't been initialized yet. */ 930473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris return (arena_tdata_get_hard(tsd, ind)); 93196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris } 932473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris if (unlikely(ind >= tsd_narenas_tdata_get(tsd))) { 93396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris /* 934473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris * ind is invalid, cache is old (too small), or tdata to be 93596d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * initialized. 93696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris */ 937473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris return (refresh_if_missing ? arena_tdata_get_hard(tsd, ind) : 938473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris NULL); 93996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris } 940473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris 941473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris tdata = &arenas_tdata[ind]; 942473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris if (likely(tdata != NULL) || !refresh_if_missing) 943473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris return (tdata); 944473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris return (arena_tdata_get_hard(tsd, ind)); 945473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris} 946473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris 947473a1853074261bb265060e04deeb8efb0fe6facChristopher FerrisJEMALLOC_INLINE arena_t * 948354524779686d6d5518078fd681110157631c941Christopher Ferrisarena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing) 949473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris{ 950473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris arena_t *ret; 951473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris 952473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris assert(ind <= MALLOCX_ARENA_MAX); 953473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris 954473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris ret = arenas[ind]; 955473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris if (unlikely(ret == NULL)) { 956473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris ret = atomic_read_p((void *)&arenas[ind]); 957473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris if (init_if_missing && unlikely(ret == NULL)) 958354524779686d6d5518078fd681110157631c941Christopher Ferris ret = arena_init(tsdn, ind); 959473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris } 960473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris return (ret); 961473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris} 962473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris 963473a1853074261bb265060e04deeb8efb0fe6facChristopher FerrisJEMALLOC_INLINE ticker_t * 964473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferrisdecay_ticker_get(tsd_t *tsd, unsigned ind) 965473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris{ 966473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris arena_tdata_t *tdata; 967473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris 968473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris tdata = arena_tdata_get(tsd, ind, true); 969473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris if (unlikely(tdata == NULL)) 970473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris return (NULL); 971473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris return (&tdata->decay_ticker); 97296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris} 9735daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 9745daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 9755daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/bitmap.h" 9765daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris/* 97796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * Include portions of arena.h interleaved with tcache.h in order to resolve 97896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * circular dependencies. 9795daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris */ 98096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#define JEMALLOC_ARENA_INLINE_A 9815daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/arena.h" 98296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#undef JEMALLOC_ARENA_INLINE_A 98353372f270c36cc1770954fa60e274bcd97c81073Nicolas Geoffray#include "jemalloc/internal/tcache.h" 98496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#define JEMALLOC_ARENA_INLINE_B 98553372f270c36cc1770954fa60e274bcd97c81073Nicolas Geoffray#include "jemalloc/internal/arena.h" 98696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris#undef JEMALLOC_ARENA_INLINE_B 9875daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/hash.h" 9885daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/quarantine.h" 9895daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 9905daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#ifndef JEMALLOC_ENABLE_INLINE 99196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisarena_t *iaalloc(const void *ptr); 992354524779686d6d5518078fd681110157631c941Christopher Ferrissize_t isalloc(tsdn_t *tsdn, const void *ptr, bool demote); 993354524779686d6d5518078fd681110157631c941Christopher Ferrisvoid *iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, 994ad6417c653ea7eeaebe6a0d56a31aa1cf1c93336Christopher Ferris tcache_t *tcache, bool is_metadata, arena_t *arena, bool slow_path); 995354524779686d6d5518078fd681110157631c941Christopher Ferrisvoid *ialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero, 996354524779686d6d5518078fd681110157631c941Christopher Ferris bool slow_path); 997354524779686d6d5518078fd681110157631c941Christopher Ferrisvoid *ipallocztm(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, 99896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris tcache_t *tcache, bool is_metadata, arena_t *arena); 999354524779686d6d5518078fd681110157631c941Christopher Ferrisvoid *ipalloct(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, 100096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris tcache_t *tcache, arena_t *arena); 100196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisvoid *ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero); 1002354524779686d6d5518078fd681110157631c941Christopher Ferrissize_t ivsalloc(tsdn_t *tsdn, const void *ptr, bool demote); 10035daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrissize_t u2rz(size_t usize); 1004354524779686d6d5518078fd681110157631c941Christopher Ferrissize_t p2rz(tsdn_t *tsdn, const void *ptr); 1005354524779686d6d5518078fd681110157631c941Christopher Ferrisvoid idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool is_metadata, 1006ad6417c653ea7eeaebe6a0d56a31aa1cf1c93336Christopher Ferris bool slow_path); 100796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisvoid idalloc(tsd_t *tsd, void *ptr); 1008ad6417c653ea7eeaebe6a0d56a31aa1cf1c93336Christopher Ferrisvoid iqalloc(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path); 1009354524779686d6d5518078fd681110157631c941Christopher Ferrisvoid isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, 1010354524779686d6d5518078fd681110157631c941Christopher Ferris bool slow_path); 1011354524779686d6d5518078fd681110157631c941Christopher Ferrisvoid isqalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache, 1012354524779686d6d5518078fd681110157631c941Christopher Ferris bool slow_path); 101396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisvoid *iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, 101496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris size_t extra, size_t alignment, bool zero, tcache_t *tcache, 10155daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris arena_t *arena); 101696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisvoid *iralloct(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, 101796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris size_t alignment, bool zero, tcache_t *tcache, arena_t *arena); 101896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisvoid *iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, 101996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris size_t alignment, bool zero); 1020354524779686d6d5518078fd681110157631c941Christopher Ferrisbool ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, 1021473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris size_t extra, size_t alignment, bool zero); 10225daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 10235daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 10245daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) 102596d58c8575eeac2a4a026e4d3629388573335bfaChristopher FerrisJEMALLOC_ALWAYS_INLINE arena_t * 102696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisiaalloc(const void *ptr) 102796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris{ 102896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 102996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris assert(ptr != NULL); 103096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 103196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris return (arena_aalloc(ptr)); 103296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris} 103396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 103496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris/* 103596d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * Typical usage: 1036354524779686d6d5518078fd681110157631c941Christopher Ferris * tsdn_t *tsdn = [...] 103796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris * void *ptr = [...] 1038354524779686d6d5518078fd681110157631c941Christopher Ferris * size_t sz = isalloc(tsdn, ptr, config_prof); 103996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris */ 104096d58c8575eeac2a4a026e4d3629388573335bfaChristopher FerrisJEMALLOC_ALWAYS_INLINE size_t 1041354524779686d6d5518078fd681110157631c941Christopher Ferrisisalloc(tsdn_t *tsdn, const void *ptr, bool demote) 104296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris{ 104396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 104496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris assert(ptr != NULL); 104596d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris /* Demotion only makes sense if config_prof is true. */ 104696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris assert(config_prof || !demote); 104796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 1048354524779686d6d5518078fd681110157631c941Christopher Ferris return (arena_salloc(tsdn, ptr, demote)); 104996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris} 105096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 10515daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher FerrisJEMALLOC_ALWAYS_INLINE void * 1052354524779686d6d5518078fd681110157631c941Christopher Ferrisiallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, tcache_t *tcache, 1053ad6417c653ea7eeaebe6a0d56a31aa1cf1c93336Christopher Ferris bool is_metadata, arena_t *arena, bool slow_path) 10545daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris{ 105596d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris void *ret; 10565daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 10575daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris assert(size != 0); 1058354524779686d6d5518078fd681110157631c941Christopher Ferris assert(!is_metadata || tcache == NULL); 1059354524779686d6d5518078fd681110157631c941Christopher Ferris assert(!is_metadata || arena == NULL || arena->ind < narenas_auto); 10605daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 1061354524779686d6d5518078fd681110157631c941Christopher Ferris ret = arena_malloc(tsdn, arena, size, ind, zero, tcache, slow_path); 106296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris if (config_stats && is_metadata && likely(ret != NULL)) { 1063354524779686d6d5518078fd681110157631c941Christopher Ferris arena_metadata_allocated_add(iaalloc(ret), 1064354524779686d6d5518078fd681110157631c941Christopher Ferris isalloc(tsdn, ret, config_prof)); 106596d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris } 106696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris return (ret); 10675daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris} 10685daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 10695daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher FerrisJEMALLOC_ALWAYS_INLINE void * 1070354524779686d6d5518078fd681110157631c941Christopher Ferrisialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero, bool slow_path) 10715daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris{ 10725daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 1073354524779686d6d5518078fd681110157631c941Christopher Ferris return (iallocztm(tsd_tsdn(tsd), size, ind, zero, tcache_get(tsd, true), 1074354524779686d6d5518078fd681110157631c941Christopher Ferris false, NULL, slow_path)); 10755daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris} 10765daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 10775daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher FerrisJEMALLOC_ALWAYS_INLINE void * 1078354524779686d6d5518078fd681110157631c941Christopher Ferrisipallocztm(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, 107996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris tcache_t *tcache, bool is_metadata, arena_t *arena) 10805daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris{ 10815daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris void *ret; 10825daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 10835daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris assert(usize != 0); 10845daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris assert(usize == sa2u(usize, alignment)); 1085354524779686d6d5518078fd681110157631c941Christopher Ferris assert(!is_metadata || tcache == NULL); 1086354524779686d6d5518078fd681110157631c941Christopher Ferris assert(!is_metadata || arena == NULL || arena->ind < narenas_auto); 10875daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 1088354524779686d6d5518078fd681110157631c941Christopher Ferris ret = arena_palloc(tsdn, arena, usize, alignment, zero, tcache); 108953372f270c36cc1770954fa60e274bcd97c81073Nicolas Geoffray assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret); 109096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris if (config_stats && is_metadata && likely(ret != NULL)) { 1091354524779686d6d5518078fd681110157631c941Christopher Ferris arena_metadata_allocated_add(iaalloc(ret), isalloc(tsdn, ret, 109296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris config_prof)); 109396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris } 10945daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris return (ret); 10955daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris} 10965daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 10975daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher FerrisJEMALLOC_ALWAYS_INLINE void * 1098354524779686d6d5518078fd681110157631c941Christopher Ferrisipalloct(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, 109996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris tcache_t *tcache, arena_t *arena) 11005daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris{ 11015daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 1102354524779686d6d5518078fd681110157631c941Christopher Ferris return (ipallocztm(tsdn, usize, alignment, zero, tcache, false, arena)); 11035daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris} 11045daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 110596d58c8575eeac2a4a026e4d3629388573335bfaChristopher FerrisJEMALLOC_ALWAYS_INLINE void * 110696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero) 11075daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris{ 11085daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 1109354524779686d6d5518078fd681110157631c941Christopher Ferris return (ipallocztm(tsd_tsdn(tsd), usize, alignment, zero, 1110354524779686d6d5518078fd681110157631c941Christopher Ferris tcache_get(tsd, true), false, NULL)); 11115daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris} 11125daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 11135daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher FerrisJEMALLOC_ALWAYS_INLINE size_t 1114354524779686d6d5518078fd681110157631c941Christopher Ferrisivsalloc(tsdn_t *tsdn, const void *ptr, bool demote) 11155daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris{ 111696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris extent_node_t *node; 11175daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 11185daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris /* Return 0 if ptr is not within a chunk managed by jemalloc. */ 1119cb4217087d4d5b1271f8316515f5e71f087b3162Christopher Ferris node = chunk_lookup(ptr, false); 112096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris if (node == NULL) 11215daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris return (0); 112296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris /* Only arena chunks should be looked up via interior pointers. */ 112396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris assert(extent_node_addr_get(node) == ptr || 112496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris extent_node_achunk_get(node)); 11255daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 1126354524779686d6d5518078fd681110157631c941Christopher Ferris return (isalloc(tsdn, ptr, demote)); 11275daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris} 11285daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 11295daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher FerrisJEMALLOC_INLINE size_t 11305daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferrisu2rz(size_t usize) 11315daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris{ 11325daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris size_t ret; 11335daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 11345daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris if (usize <= SMALL_MAXCLASS) { 11356f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris szind_t binind = size2index(usize); 11365daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris ret = arena_bin_info[binind].redzone_size; 11375daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris } else 11385daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris ret = 0; 11395daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 11405daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris return (ret); 11415daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris} 11425daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 11435daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher FerrisJEMALLOC_INLINE size_t 1144354524779686d6d5518078fd681110157631c941Christopher Ferrisp2rz(tsdn_t *tsdn, const void *ptr) 11455daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris{ 1146354524779686d6d5518078fd681110157631c941Christopher Ferris size_t usize = isalloc(tsdn, ptr, false); 11475daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 11485daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris return (u2rz(usize)); 11495daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris} 11505daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 11515daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher FerrisJEMALLOC_ALWAYS_INLINE void 1152354524779686d6d5518078fd681110157631c941Christopher Ferrisidalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool is_metadata, 1153ad6417c653ea7eeaebe6a0d56a31aa1cf1c93336Christopher Ferris bool slow_path) 11545daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris{ 11555daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 11565daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris assert(ptr != NULL); 1157354524779686d6d5518078fd681110157631c941Christopher Ferris assert(!is_metadata || tcache == NULL); 1158354524779686d6d5518078fd681110157631c941Christopher Ferris assert(!is_metadata || iaalloc(ptr)->ind < narenas_auto); 115996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris if (config_stats && is_metadata) { 1160354524779686d6d5518078fd681110157631c941Christopher Ferris arena_metadata_allocated_sub(iaalloc(ptr), isalloc(tsdn, ptr, 116196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris config_prof)); 116296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris } 11630b9d74bfed176f3c83114086a9dbf207cdeeb1f1Christopher Ferris 1164354524779686d6d5518078fd681110157631c941Christopher Ferris arena_dalloc(tsdn, ptr, tcache, slow_path); 11655daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris} 11665daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 11675daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher FerrisJEMALLOC_ALWAYS_INLINE void 116896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisidalloc(tsd_t *tsd, void *ptr) 11695daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris{ 11705daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 1171354524779686d6d5518078fd681110157631c941Christopher Ferris idalloctm(tsd_tsdn(tsd), ptr, tcache_get(tsd, false), false, true); 11725daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris} 11735daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 11745daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher FerrisJEMALLOC_ALWAYS_INLINE void 1175ad6417c653ea7eeaebe6a0d56a31aa1cf1c93336Christopher Ferrisiqalloc(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) 11765daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris{ 11775daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 1178ad6417c653ea7eeaebe6a0d56a31aa1cf1c93336Christopher Ferris if (slow_path && config_fill && unlikely(opt_quarantine)) 117996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris quarantine(tsd, ptr); 11805daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris else 1181354524779686d6d5518078fd681110157631c941Christopher Ferris idalloctm(tsd_tsdn(tsd), ptr, tcache, false, slow_path); 11825daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris} 11835daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 11845daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher FerrisJEMALLOC_ALWAYS_INLINE void 1185354524779686d6d5518078fd681110157631c941Christopher Ferrisisdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, 1186354524779686d6d5518078fd681110157631c941Christopher Ferris bool slow_path) 11875daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris{ 11885daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 1189354524779686d6d5518078fd681110157631c941Christopher Ferris arena_sdalloc(tsdn, ptr, size, tcache, slow_path); 119096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris} 119196d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 119296d58c8575eeac2a4a026e4d3629388573335bfaChristopher FerrisJEMALLOC_ALWAYS_INLINE void 1193354524779686d6d5518078fd681110157631c941Christopher Ferrisisqalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache, bool slow_path) 119496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris{ 119596d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris 1196354524779686d6d5518078fd681110157631c941Christopher Ferris if (slow_path && config_fill && unlikely(opt_quarantine)) 119796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris quarantine(tsd, ptr); 119896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris else 1199354524779686d6d5518078fd681110157631c941Christopher Ferris isdalloct(tsd_tsdn(tsd), ptr, size, tcache, slow_path); 12005daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris} 12015daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 12025daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher FerrisJEMALLOC_ALWAYS_INLINE void * 120396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisiralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, 120496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris size_t extra, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena) 12055daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris{ 12065daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris void *p; 12075daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris size_t usize, copysize; 12085daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 12095daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris usize = sa2u(size + extra, alignment); 1210473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) 12115daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris return (NULL); 1212354524779686d6d5518078fd681110157631c941Christopher Ferris p = ipalloct(tsd_tsdn(tsd), usize, alignment, zero, tcache, arena); 12135daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris if (p == NULL) { 12145daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris if (extra == 0) 12155daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris return (NULL); 12165daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris /* Try again, without extra this time. */ 12175daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris usize = sa2u(size, alignment); 1218473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) 12195daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris return (NULL); 1220354524779686d6d5518078fd681110157631c941Christopher Ferris p = ipalloct(tsd_tsdn(tsd), usize, alignment, zero, tcache, 1221354524779686d6d5518078fd681110157631c941Christopher Ferris arena); 12225daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris if (p == NULL) 12235daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris return (NULL); 12245daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris } 12255daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris /* 12265daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * Copy at most size bytes (not size+extra), since the caller has no 12275daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * expectation that the extra bytes will be reliably preserved. 12285daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris */ 12295daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris copysize = (size < oldsize) ? size : oldsize; 12305daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris memcpy(p, ptr, copysize); 1231354524779686d6d5518078fd681110157631c941Christopher Ferris isqalloc(tsd, ptr, oldsize, tcache, true); 12325daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris return (p); 12335daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris} 12345daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 12355daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher FerrisJEMALLOC_ALWAYS_INLINE void * 123696d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisiralloct(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment, 123796d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris bool zero, tcache_t *tcache, arena_t *arena) 12385daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris{ 12395daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 12405daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris assert(ptr != NULL); 12415daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris assert(size != 0); 12425daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 12435daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) 12445daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris != 0) { 12455daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris /* 12465daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * Existing object alignment is inadequate; allocate new space 12475daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris * and copy. 12485daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris */ 124996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris return (iralloct_realign(tsd, ptr, oldsize, size, 0, alignment, 125096d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris zero, tcache, arena)); 12515daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris } 12525daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 1253ad6417c653ea7eeaebe6a0d56a31aa1cf1c93336Christopher Ferris return (arena_ralloc(tsd, arena, ptr, oldsize, size, alignment, zero, 125496d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris tcache)); 12555daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris} 12565daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 12575daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher FerrisJEMALLOC_ALWAYS_INLINE void * 125896d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferrisiralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment, 125996d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris bool zero) 12605daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris{ 12615daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 126296d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris return (iralloct(tsd, ptr, oldsize, size, alignment, zero, 126396d58c8575eeac2a4a026e4d3629388573335bfaChristopher Ferris tcache_get(tsd, true), NULL)); 12645daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris} 12655daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 12665daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher FerrisJEMALLOC_ALWAYS_INLINE bool 1267354524779686d6d5518078fd681110157631c941Christopher Ferrisixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra, 1268473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris size_t alignment, bool zero) 12695daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris{ 12705daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 12715daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris assert(ptr != NULL); 12725daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris assert(size != 0); 12735daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 12745daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) 12755daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris != 0) { 12765daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris /* Existing object alignment is inadequate. */ 12775daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris return (true); 12785daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris } 12795daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 1280354524779686d6d5518078fd681110157631c941Christopher Ferris return (arena_ralloc_no_move(tsdn, ptr, oldsize, size, extra, zero)); 12815daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris} 12825daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif 12835daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 12845daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#include "jemalloc/internal/prof.h" 12855daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris 12865daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#undef JEMALLOC_H_INLINES 12875daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris/******************************************************************************/ 12885daf4e4a8d52ac2d5b40b0d12ce5721c6b9676e7Christopher Ferris#endif /* JEMALLOC_INTERNAL_H */ 1289