jemalloc_internal.h.in revision 7427525c28d58c423a68930160e3b0fe577fe953
149837ef8111fbeace7ae6379ca733c8f8fa94cfeChandler Carruth#include <sys/mman.h> 2ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include <sys/param.h> 3ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include <sys/time.h> 4ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include <sys/types.h> 57ed47a13356daed2a34cd2209a31f92552e3bdd8Chris Lattner#include <sys/sysctl.h> 67ed47a13356daed2a34cd2209a31f92552e3bdd8Chris Lattner#include <sys/uio.h> 7ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel 8ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include <errno.h> 9ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include <limits.h> 1019c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick#ifndef SIZE_T_MAX 11ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel# define SIZE_T_MAX SIZE_MAX 12ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#endif 13ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include <pthread.h> 14d68a07650cdb2e18f18f362ba533459aa10e01b6Dan Gohman#include <sched.h> 15d68a07650cdb2e18f18f362ba533459aa10e01b6Dan Gohman#include <stdarg.h> 16f522068412218cd14b2c2df74a3437717d255381Dan Gohman#include <stdbool.h> 17c6db6b6f26907f00a4b1665030616f90dbb8f546Bill Wendling#include <stdio.h> 183b8a90686adb3d0adaa4389e7b7900570a235e03Devang Patel#include <stdlib.h> 19255f89faee13dc491cb64fbeae3c763e7e2ea4e6Chandler Carruth#include <stdint.h> 20255f89faee13dc491cb64fbeae3c763e7e2ea4e6Chandler Carruth#include <stddef.h> 21255f89faee13dc491cb64fbeae3c763e7e2ea4e6Chandler Carruth#ifndef offsetof 22c9235d2e855c56e9aa157969f8132a05f9ba89d8Dan Gohman# define offsetof(type, member) ((size_t)&(((type *)NULL)->member)) 23255f89faee13dc491cb64fbeae3c763e7e2ea4e6Chandler Carruth#endif 24ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include <inttypes.h> 25ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include <string.h> 26ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include <strings.h> 27ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include <ctype.h> 2819c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick#include <unistd.h> 29ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include <fcntl.h> 30ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include <pthread.h> 31ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include <math.h> 3219c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick 33ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#define JEMALLOC_MANGLE 34ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "../jemalloc@install_suffix@.h" 35ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel 36ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN)) 37e6acf36dca9864c6ead9113f0322f174469810ccDan Gohman#include <libkern/OSAtomic.h> 38e6acf36dca9864c6ead9113f0322f174469810ccDan Gohman#endif 39ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel 40ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#ifdef JEMALLOC_ZONE 41ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include <mach/mach_error.h> 42ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include <mach/mach_init.h> 43ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include <mach/vm_map.h> 44ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include <malloc/malloc.h> 45ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#endif 46ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel 4719c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick#ifdef JEMALLOC_LAZY_LOCK 4819c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick#include <dlfcn.h> 49ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#endif 50ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel 51ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#define RB_COMPACT 52ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/rb.h" 5319c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick#include "jemalloc/internal/qr.h" 54ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/ql.h" 55ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel 56ab7752c1496c2913793305ba4b989a551c5617e1Devang Patelextern void (*JEMALLOC_P(malloc_message))(void *wcbopaque, const char *s); 57ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel 58ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel/* 59ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel * Define a custom assert() in order to reduce the chances of deadlock during 60ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel * assertion failure. 61ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel */ 62ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#ifndef assert 63ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel# ifdef JEMALLOC_DEBUG 64ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel# define assert(e) do { \ 65ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel if (!(e)) { \ 66ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel char line_buf[UMAX2S_BUFSIZE]; \ 67ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel malloc_write("<jemalloc>: "); \ 68ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel malloc_write(__FILE__); \ 69ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel malloc_write(":"); \ 70ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel malloc_write(u2s(__LINE__, 10, line_buf)); \ 71ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel malloc_write(": Failed assertion: "); \ 72ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel malloc_write("\""); \ 73ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel malloc_write(#e); \ 74ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel malloc_write("\"\n"); \ 75ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel abort(); \ 76ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel } \ 77ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel} while (0) 78ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel# else 79ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel# define assert(e) 80ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel# endif 81ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#endif 82ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel 83ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#ifdef JEMALLOC_DEBUG 84ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel# define dassert(e) assert(e) 85ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#else 8611e43291540db9d885b736cbd652558faab80955Andrew Trick# define dassert(e) 87ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#endif 88ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel 89ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel/* 90ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel * jemalloc can conceptually be broken into components (arena, tcache, etc.), 91ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel * but there are circular dependencies that cannot be broken without 92d6f16587ab292b20857933b3ba13d3d1c62a8d62Chris Lattner * substantial performance degradation. In order to reduce the effect on 93d6f16587ab292b20857933b3ba13d3d1c62a8d62Chris Lattner * visual code flow, read the header files in multiple passes, with one of the 94ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel * following cpp variables defined during each pass: 9593b67e40de356569493c285b86b138a3f11b5035Daniel Dunbar * 96d6f16587ab292b20857933b3ba13d3d1c62a8d62Chris Lattner * JEMALLOC_H_TYPES : Preprocessor-defined constants and psuedo-opaque data 9793b67e40de356569493c285b86b138a3f11b5035Daniel Dunbar * types. 98d6f16587ab292b20857933b3ba13d3d1c62a8d62Chris Lattner * JEMALLOC_H_STRUCTS : Data structures. 995c12adaa8b92800d835ddd5c97723c6eb5628b5bDan Gohman * JEMALLOC_H_EXTERNS : Extern data declarations and function prototypes. 1004afefdec65f198fad803fa8f9f994663488dd608Dan Gohman * JEMALLOC_H_INLINES : Inline functions. 101ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel */ 1027f99761143cdf1ad23729708e3dcaa055b189406Devang Patel/******************************************************************************/ 1037f99761143cdf1ad23729708e3dcaa055b189406Devang Patel#define JEMALLOC_H_TYPES 10436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 10536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#define ALLOCM_LG_ALIGN_MASK ((int)0x3f) 10636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 10736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#define ZU(z) ((size_t)z) 1087f99761143cdf1ad23729708e3dcaa055b189406Devang Patel 1097f99761143cdf1ad23729708e3dcaa055b189406Devang Patel#ifndef __DECONST 11036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines# define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var)) 11136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#endif 11236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 11319c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick#ifdef JEMALLOC_DEBUG 1147f99761143cdf1ad23729708e3dcaa055b189406Devang Patel /* Disable inlining to make debugging easier. */ 115312e23485b4dcaf9b7c4956961bda4423bf6d73eChris Lattner# define JEMALLOC_INLINE 116312e23485b4dcaf9b7c4956961bda4423bf6d73eChris Lattner# define inline 117d6f16587ab292b20857933b3ba13d3d1c62a8d62Chris Lattner#else 118d6f16587ab292b20857933b3ba13d3d1c62a8d62Chris Lattner# define JEMALLOC_ENABLE_INLINE 119d6f16587ab292b20857933b3ba13d3d1c62a8d62Chris Lattner# define JEMALLOC_INLINE static inline 120d6f16587ab292b20857933b3ba13d3d1c62a8d62Chris Lattner#endif 121d6f16587ab292b20857933b3ba13d3d1c62a8d62Chris Lattner 122d87dc0fdabf8549fadb5cba0b20f36310614f101Dan Gohman/* Size of stack-allocated buffer passed to buferror(). */ 123dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#define BUFERROR_BUF 64 124d6f16587ab292b20857933b3ba13d3d1c62a8d62Chris Lattner 125dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/* Minimum alignment of allocations is 2^LG_QUANTUM bytes. */ 126d6f16587ab292b20857933b3ba13d3d1c62a8d62Chris Lattner#ifdef __i386__ 127dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines# define LG_QUANTUM 4 12819c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick#endif 129d6f16587ab292b20857933b3ba13d3d1c62a8d62Chris Lattner#ifdef __ia64__ 13036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines# define LG_QUANTUM 4 131d6f16587ab292b20857933b3ba13d3d1c62a8d62Chris Lattner#endif 13219c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick#ifdef __alpha__ 13319c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick# define LG_QUANTUM 4 134ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#endif 135d7a3541a059d8b2b17b67ea4e8d2bfaada6d4bdbDan Gohman#ifdef __sparc64__ 136d7a3541a059d8b2b17b67ea4e8d2bfaada6d4bdbDan Gohman# define LG_QUANTUM 4 1374bdeede8c7ea307b3f6192ec4f425debda948998Dan Gohman#endif 1384bdeede8c7ea307b3f6192ec4f425debda948998Dan Gohman#if (defined(__amd64__) || defined(__x86_64__)) 1394bdeede8c7ea307b3f6192ec4f425debda948998Dan Gohman# define LG_QUANTUM 4 14019c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick#endif 141d7a3541a059d8b2b17b67ea4e8d2bfaada6d4bdbDan Gohman#ifdef __arm__ 142d7a3541a059d8b2b17b67ea4e8d2bfaada6d4bdbDan Gohman# define LG_QUANTUM 3 143d7a3541a059d8b2b17b67ea4e8d2bfaada6d4bdbDan Gohman#endif 144d7a3541a059d8b2b17b67ea4e8d2bfaada6d4bdbDan Gohman#ifdef __mips__ 145d7a3541a059d8b2b17b67ea4e8d2bfaada6d4bdbDan Gohman# define LG_QUANTUM 3 1464bdeede8c7ea307b3f6192ec4f425debda948998Dan Gohman#endif 1474bdeede8c7ea307b3f6192ec4f425debda948998Dan Gohman#ifdef __powerpc__ 1484bdeede8c7ea307b3f6192ec4f425debda948998Dan Gohman# define LG_QUANTUM 4 149d7a3541a059d8b2b17b67ea4e8d2bfaada6d4bdbDan Gohman#endif 150d7a3541a059d8b2b17b67ea4e8d2bfaada6d4bdbDan Gohman#ifdef __s390x__ 1514bdeede8c7ea307b3f6192ec4f425debda948998Dan Gohman# define LG_QUANTUM 4 152d7a3541a059d8b2b17b67ea4e8d2bfaada6d4bdbDan Gohman#endif 1534bdeede8c7ea307b3f6192ec4f425debda948998Dan Gohman 1544bdeede8c7ea307b3f6192ec4f425debda948998Dan Gohman#define QUANTUM ((size_t)(1U << LG_QUANTUM)) 1554bdeede8c7ea307b3f6192ec4f425debda948998Dan Gohman#define QUANTUM_MASK (QUANTUM - 1) 156d7a3541a059d8b2b17b67ea4e8d2bfaada6d4bdbDan Gohman 157d7a3541a059d8b2b17b67ea4e8d2bfaada6d4bdbDan Gohman/* Return the smallest quantum multiple that is >= a. */ 1584bdeede8c7ea307b3f6192ec4f425debda948998Dan Gohman#define QUANTUM_CEILING(a) \ 159d7a3541a059d8b2b17b67ea4e8d2bfaada6d4bdbDan Gohman (((a) + QUANTUM_MASK) & ~QUANTUM_MASK) 160d7a3541a059d8b2b17b67ea4e8d2bfaada6d4bdbDan Gohman 161d7a3541a059d8b2b17b67ea4e8d2bfaada6d4bdbDan Gohman#define LONG ((size_t)(1U << LG_SIZEOF_LONG)) 162d7a3541a059d8b2b17b67ea4e8d2bfaada6d4bdbDan Gohman#define LONG_MASK (LONG - 1) 163ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel 164ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel/* Return the smallest long multiple that is >= a. */ 165ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#define LONG_CEILING(a) \ 166ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel (((a) + LONG_MASK) & ~LONG_MASK) 167ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel 1689fb7d04cfcfd3e206001f0eea17c69afcdcc0080Dan Gohman#define SIZEOF_PTR (1U << LG_SIZEOF_PTR) 1699fb7d04cfcfd3e206001f0eea17c69afcdcc0080Dan Gohman#define PTR_MASK (SIZEOF_PTR - 1) 170ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel 1714712b804dfe02bc7bbf948f9a5e352b3f14cc89aEvan Cheng/* Return the smallest (void *) multiple that is >= a. */ 17234cd4a484e532cc463fd5a4bf59b88d13c5467c1Evan Cheng#define PTR_CEILING(a) \ 173ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel (((a) + PTR_MASK) & ~PTR_MASK) 174ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel 1759fb7d04cfcfd3e206001f0eea17c69afcdcc0080Dan Gohman/* 176ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel * Maximum size of L1 cache line. This is used to avoid cache line aliasing. 1779fb7d04cfcfd3e206001f0eea17c69afcdcc0080Dan Gohman * In addition, this controls the spacing of cacheline-spaced size classes. 17811e43291540db9d885b736cbd652558faab80955Andrew Trick */ 17911e43291540db9d885b736cbd652558faab80955Andrew Trick#define LG_CACHELINE 6 180ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#define CACHELINE ((size_t)(1U << LG_CACHELINE)) 1819fb7d04cfcfd3e206001f0eea17c69afcdcc0080Dan Gohman#define CACHELINE_MASK (CACHELINE - 1) 1829fb7d04cfcfd3e206001f0eea17c69afcdcc0080Dan Gohman 1839fb7d04cfcfd3e206001f0eea17c69afcdcc0080Dan Gohman/* Return the smallest cacheline multiple that is >= s. */ 1849fb7d04cfcfd3e206001f0eea17c69afcdcc0080Dan Gohman#define CACHELINE_CEILING(s) \ 1859fb7d04cfcfd3e206001f0eea17c69afcdcc0080Dan Gohman (((s) + CACHELINE_MASK) & ~CACHELINE_MASK) 1869fb7d04cfcfd3e206001f0eea17c69afcdcc0080Dan Gohman 187ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel/* 188c6db6b6f26907f00a4b1665030616f90dbb8f546Bill Wendling * Page size. STATIC_PAGE_SHIFT is determined by the configure script. If 189ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel * DYNAMIC_PAGE_SHIFT is enabled, only use the STATIC_PAGE_* macros where 190ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel * compile-time values are required for the purposes of defining data 191ebb1834e86d8da5fe7c63986bf8f350574a40a6cDan Gohman * structures. 192ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel */ 193ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#define STATIC_PAGE_SIZE ((size_t)(1U << STATIC_PAGE_SHIFT)) 194ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#define STATIC_PAGE_MASK ((size_t)(STATIC_PAGE_SIZE - 1)) 195ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel 196ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#ifdef PAGE_SHIFT 197ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel# undef PAGE_SHIFT 1983b8a90686adb3d0adaa4389e7b7900570a235e03Devang Patel#endif 1993b8a90686adb3d0adaa4389e7b7900570a235e03Devang Patel#ifdef PAGE_SIZE 2003b8a90686adb3d0adaa4389e7b7900570a235e03Devang Patel# undef PAGE_SIZE 20119c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick#endif 202ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#ifdef PAGE_MASK 203ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel# undef PAGE_MASK 204ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#endif 205ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel 206ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#ifdef DYNAMIC_PAGE_SHIFT 207ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel# define PAGE_SHIFT lg_pagesize 208ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel# define PAGE_SIZE pagesize 209ebb1834e86d8da5fe7c63986bf8f350574a40a6cDan Gohman# define PAGE_MASK pagesize_mask 210ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#else 211ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel# define PAGE_SHIFT STATIC_PAGE_SHIFT 212ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel# define PAGE_SIZE STATIC_PAGE_SIZE 2130961ec1afe336a0a4f523455c25f5d5edc8d3ee1Dan Gohman# define PAGE_MASK STATIC_PAGE_MASK 214ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#endif 215ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel 216ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel/* Return the smallest pagesize multiple that is >= s. */ 217ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#define PAGE_CEILING(s) \ 218ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel (((s) + PAGE_MASK) & ~PAGE_MASK) 219ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel 220ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/atomic.h" 221ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/prn.h" 222ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/ckh.h" 223ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/stats.h" 224ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/ctl.h" 225ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/mutex.h" 226ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/mb.h" 227ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/extent.h" 228ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/arena.h" 229ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/bitmap.h" 230ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/base.h" 23119c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick#include "jemalloc/internal/chunk.h" 232ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/huge.h" 23378766ff67cb9222a2dd5e2cda2e1488b5288a79dDevang Patel#include "jemalloc/internal/rtree.h" 234ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/tcache.h" 235ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/hash.h" 236ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#ifdef JEMALLOC_ZONE 237ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/zone.h" 238ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#endif 23978766ff67cb9222a2dd5e2cda2e1488b5288a79dDevang Patel#include "jemalloc/internal/prof.h" 240ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel 241ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#undef JEMALLOC_H_TYPES 242ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel/******************************************************************************/ 243721e59cfb29024ffad7204f05ad75b678ae7df63Devang Patel#define JEMALLOC_H_STRUCTS 244721e59cfb29024ffad7204f05ad75b678ae7df63Devang Patel 245721e59cfb29024ffad7204f05ad75b678ae7df63Devang Patel#include "jemalloc/internal/atomic.h" 246721e59cfb29024ffad7204f05ad75b678ae7df63Devang Patel#include "jemalloc/internal/prn.h" 247721e59cfb29024ffad7204f05ad75b678ae7df63Devang Patel#include "jemalloc/internal/ckh.h" 248721e59cfb29024ffad7204f05ad75b678ae7df63Devang Patel#include "jemalloc/internal/stats.h" 249ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/ctl.h" 250ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/mutex.h" 25178766ff67cb9222a2dd5e2cda2e1488b5288a79dDevang Patel#include "jemalloc/internal/mb.h" 2523b8a90686adb3d0adaa4389e7b7900570a235e03Devang Patel#include "jemalloc/internal/bitmap.h" 2533b8a90686adb3d0adaa4389e7b7900570a235e03Devang Patel#include "jemalloc/internal/extent.h" 254ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/arena.h" 255ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/base.h" 256ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/chunk.h" 25719c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick#include "jemalloc/internal/huge.h" 258ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/rtree.h" 259ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/tcache.h" 260ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/hash.h" 261ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#ifdef JEMALLOC_ZONE 262ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/zone.h" 263ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#endif 264ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/prof.h" 265fe613905b371ef7160968cdaabf6074cfa10f4a0Devang Patel 266dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#ifdef JEMALLOC_STATS 267ab7752c1496c2913793305ba4b989a551c5617e1Devang Pateltypedef struct { 268ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel uint64_t allocated; 269ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel uint64_t deallocated; 270ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel} thread_allocated_t; 27119c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick#endif 2723660ecabbb85b31308f38938ce3f56f0a330a84bChris Lattner 273ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#undef JEMALLOC_H_STRUCTS 274ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel/******************************************************************************/ 275ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#define JEMALLOC_H_EXTERNS 276ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel 27758e0ef1e90c3f6dbae213612b44e56f7d6d65ea7Devang Patelextern bool opt_abort; 27858e0ef1e90c3f6dbae213612b44e56f7d6d65ea7Devang Patel#ifdef JEMALLOC_FILL 27958e0ef1e90c3f6dbae213612b44e56f7d6d65ea7Devang Patelextern bool opt_junk; 280ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#endif 281ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#ifdef JEMALLOC_SYSV 28219c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trickextern bool opt_sysv; 28327a8fb8a546ec5d39b057bfe2ee5c36a8e454f09Dan Gohman#endif 28419c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick#ifdef JEMALLOC_XMALLOC 28593b67e40de356569493c285b86b138a3f11b5035Daniel Dunbarextern bool opt_xmalloc; 286ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#endif 28727a8fb8a546ec5d39b057bfe2ee5c36a8e454f09Dan Gohman#ifdef JEMALLOC_FILL 28819c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trickextern bool opt_zero; 28927a8fb8a546ec5d39b057bfe2ee5c36a8e454f09Dan Gohman#endif 29027a8fb8a546ec5d39b057bfe2ee5c36a8e454f09Dan Gohmanextern size_t opt_narenas; 29119c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick 292ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#ifdef DYNAMIC_PAGE_SHIFT 293e24e0e1244dc273016e541948086a89ffce1f7e4Devang Patelextern size_t pagesize; 294ab7752c1496c2913793305ba4b989a551c5617e1Devang Patelextern size_t pagesize_mask; 295569a6fd9d59d67bc79f63f69ef14f91b0fda6c4aDevang Patelextern size_t lg_pagesize; 296569a6fd9d59d67bc79f63f69ef14f91b0fda6c4aDevang Patel#endif 297569a6fd9d59d67bc79f63f69ef14f91b0fda6c4aDevang Patel 29819fe8f907e4addad1a33b7590b3f05893c39b3a6Devang Patel/* Number of CPUs. */ 299569a6fd9d59d67bc79f63f69ef14f91b0fda6c4aDevang Patelextern unsigned ncpus; 30090c579de5a383cee278acc3f7e7b9d0a656e6a35Owen Anderson 301569a6fd9d59d67bc79f63f69ef14f91b0fda6c4aDevang Patelextern malloc_mutex_t arenas_lock; /* Protects arenas initialization. */ 302ab7752c1496c2913793305ba4b989a551c5617e1Devang Patelextern pthread_key_t arenas_tsd; 30319c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick#ifndef NO_TLS 304ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel/* 305fe613905b371ef7160968cdaabf6074cfa10f4a0Devang Patel * Map of pthread_self() --> arenas[???], used for selecting an arena to use 306dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines * for allocations. 307ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel */ 308ab7752c1496c2913793305ba4b989a551c5617e1Devang Patelextern __thread arena_t *arenas_tls JEMALLOC_ATTR(tls_model("initial-exec")); 3097b65dd91c156590c03847152a202bf7aa9b0c556Devang Patel# define ARENA_GET() arenas_tls 3107b65dd91c156590c03847152a202bf7aa9b0c556Devang Patel# define ARENA_SET(v) do { \ 3117b65dd91c156590c03847152a202bf7aa9b0c556Devang Patel arenas_tls = (v); \ 3127b65dd91c156590c03847152a202bf7aa9b0c556Devang Patel pthread_setspecific(arenas_tsd, (void *)(v)); \ 3137b65dd91c156590c03847152a202bf7aa9b0c556Devang Patel} while (0) 314569a6fd9d59d67bc79f63f69ef14f91b0fda6c4aDevang Patel#else 315569a6fd9d59d67bc79f63f69ef14f91b0fda6c4aDevang Patel# define ARENA_GET() ((arena_t *)pthread_getspecific(arenas_tsd)) 316569a6fd9d59d67bc79f63f69ef14f91b0fda6c4aDevang Patel# define ARENA_SET(v) do { \ 317ebb1834e86d8da5fe7c63986bf8f350574a40a6cDan Gohman pthread_setspecific(arenas_tsd, (void *)(v)); \ 318ebb1834e86d8da5fe7c63986bf8f350574a40a6cDan Gohman} while (0) 319569a6fd9d59d67bc79f63f69ef14f91b0fda6c4aDevang Patel#endif 320ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel 321ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel/* 322ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel * Arenas that are used to service external requests. Not all elements of the 323ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel * arenas array are necessarily used; arenas are created lazily as needed. 324ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel */ 325ab7752c1496c2913793305ba4b989a551c5617e1Devang Patelextern arena_t **arenas; 326ab7752c1496c2913793305ba4b989a551c5617e1Devang Patelextern unsigned narenas; 327ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel 328ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#ifdef JEMALLOC_STATS 329ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel# ifndef NO_TLS 330ab7752c1496c2913793305ba4b989a551c5617e1Devang Patelextern __thread thread_allocated_t thread_allocated_tls; 331ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel# define ALLOCATED_GET() (thread_allocated_tls.allocated) 332ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel# define ALLOCATEDP_GET() (&thread_allocated_tls.allocated) 333ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel# define DEALLOCATED_GET() (thread_allocated_tls.deallocated) 334ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel# define DEALLOCATEDP_GET() (&thread_allocated_tls.deallocated) 335ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel# define ALLOCATED_ADD(a, d) do { \ 3360e122d1c2422285c872f68fc0ae1f7e5d2739572Andrew Trick thread_allocated_tls.allocated += a; \ 337ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel thread_allocated_tls.deallocated += d; \ 338ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel} while (0) 339ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel# else 340ab7752c1496c2913793305ba4b989a551c5617e1Devang Patelextern pthread_key_t thread_allocated_tsd; 3417f99761143cdf1ad23729708e3dcaa055b189406Devang Patelthread_allocated_t *thread_allocated_get_hard(void); 3422928c83b010f7cfdb0f819199d806f6942a7d995Daniel Dunbar 3430dabb7e8f3e371cf0d6a9747b5bd4b555cc32c95Chris Lattner# define ALLOCATED_GET() (thread_allocated_get()->allocated) 3440dabb7e8f3e371cf0d6a9747b5bd4b555cc32c95Chris Lattner# define ALLOCATEDP_GET() (&thread_allocated_get()->allocated) 345ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel# define DEALLOCATED_GET() (thread_allocated_get()->deallocated) 3464712b804dfe02bc7bbf948f9a5e352b3f14cc89aEvan Cheng# define DEALLOCATEDP_GET() (&thread_allocated_get()->deallocated) 347310fa65ab9acb96319d0fcb779cdd35350f5d00fEvan Cheng# define ALLOCATED_ADD(a, d) do { \ 348ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel thread_allocated_t *thread_allocated = thread_allocated_get(); \ 349ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel thread_allocated->allocated += (a); \ 35019c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick thread_allocated->deallocated += (d); \ 351ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel} while (0) 35219c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick# endif 353ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#endif 354fe613905b371ef7160968cdaabf6074cfa10f4a0Devang Patel 355ce522ee0a27062390f13e7ccb53fcff4fc36c473Michael Ilsemanarena_t *arenas_extend(unsigned ind); 356fe613905b371ef7160968cdaabf6074cfa10f4a0Devang Patelarena_t *choose_arena_hard(void); 357fe613905b371ef7160968cdaabf6074cfa10f4a0Devang Patelint buferror(int errnum, char *buf, size_t buflen); 358fe613905b371ef7160968cdaabf6074cfa10f4a0Devang Patelvoid jemalloc_prefork(void); 359fe613905b371ef7160968cdaabf6074cfa10f4a0Devang Patelvoid jemalloc_postfork(void); 360fe613905b371ef7160968cdaabf6074cfa10f4a0Devang Patel 361fe613905b371ef7160968cdaabf6074cfa10f4a0Devang Patel#include "jemalloc/internal/atomic.h" 362fe613905b371ef7160968cdaabf6074cfa10f4a0Devang Patel#include "jemalloc/internal/prn.h" 363fe613905b371ef7160968cdaabf6074cfa10f4a0Devang Patel#include "jemalloc/internal/ckh.h" 364fe613905b371ef7160968cdaabf6074cfa10f4a0Devang Patel#include "jemalloc/internal/stats.h" 365fe613905b371ef7160968cdaabf6074cfa10f4a0Devang Patel#include "jemalloc/internal/ctl.h" 366fe613905b371ef7160968cdaabf6074cfa10f4a0Devang Patel#include "jemalloc/internal/mutex.h" 367ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/mb.h" 368ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/bitmap.h" 369ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/extent.h" 370ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/arena.h" 371ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/base.h" 372310fa65ab9acb96319d0fcb779cdd35350f5d00fEvan Cheng#include "jemalloc/internal/chunk.h" 373310fa65ab9acb96319d0fcb779cdd35350f5d00fEvan Cheng#include "jemalloc/internal/huge.h" 374310fa65ab9acb96319d0fcb779cdd35350f5d00fEvan Cheng#include "jemalloc/internal/rtree.h" 375fe613905b371ef7160968cdaabf6074cfa10f4a0Devang Patel#include "jemalloc/internal/tcache.h" 376fe613905b371ef7160968cdaabf6074cfa10f4a0Devang Patel#include "jemalloc/internal/hash.h" 377fe613905b371ef7160968cdaabf6074cfa10f4a0Devang Patel#ifdef JEMALLOC_ZONE 378ce522ee0a27062390f13e7ccb53fcff4fc36c473Michael Ilseman#include "jemalloc/internal/zone.h" 37919c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick#endif 3809554c613282c28fbe94bc8b0a5ba736f046643b8Chris Lattner#include "jemalloc/internal/prof.h" 3819554c613282c28fbe94bc8b0a5ba736f046643b8Chris Lattner 3829554c613282c28fbe94bc8b0a5ba736f046643b8Chris Lattner#undef JEMALLOC_H_EXTERNS 38319c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick/******************************************************************************/ 384ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#define JEMALLOC_H_INLINES 3852928c83b010f7cfdb0f819199d806f6942a7d995Daniel Dunbar 3862928c83b010f7cfdb0f819199d806f6942a7d995Daniel Dunbar#include "jemalloc/internal/atomic.h" 3870dabb7e8f3e371cf0d6a9747b5bd4b555cc32c95Chris Lattner#include "jemalloc/internal/prn.h" 38819c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick#include "jemalloc/internal/ckh.h" 38919c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick#include "jemalloc/internal/stats.h" 390f451cb870efcf9e0302d25ed05f4cac6bb494e42Dan Gohman#include "jemalloc/internal/ctl.h" 391ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/mutex.h" 392ce522ee0a27062390f13e7ccb53fcff4fc36c473Michael Ilseman#include "jemalloc/internal/mb.h" 393ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#include "jemalloc/internal/extent.h" 3947b65dd91c156590c03847152a202bf7aa9b0c556Devang Patel#include "jemalloc/internal/base.h" 3957b65dd91c156590c03847152a202bf7aa9b0c556Devang Patel#include "jemalloc/internal/chunk.h" 39678766ff67cb9222a2dd5e2cda2e1488b5288a79dDevang Patel#include "jemalloc/internal/huge.h" 3977b65dd91c156590c03847152a202bf7aa9b0c556Devang Patel 398ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#ifndef JEMALLOC_ENABLE_INLINE 399ab7752c1496c2913793305ba4b989a551c5617e1Devang Patelsize_t pow2_ceil(size_t x); 400ab7752c1496c2913793305ba4b989a551c5617e1Devang Patelsize_t s2u(size_t size); 401ab7752c1496c2913793305ba4b989a551c5617e1Devang Patelsize_t sa2u(size_t size, size_t alignment, size_t *run_size_p); 402ab7752c1496c2913793305ba4b989a551c5617e1Devang Patelvoid malloc_write(const char *s); 403ab7752c1496c2913793305ba4b989a551c5617e1Devang Patelarena_t *choose_arena(void); 404ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel# if (defined(JEMALLOC_STATS) && defined(NO_TLS)) 40519c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trickthread_allocated_t *thread_allocated_get(void); 40619c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick# endif 407ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#endif 408ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel 409ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) 4101997473cf72957d0e70322e2fe6fe2ab141c58a6Devang Patel/* Compute the smallest power of 2 that is >= x. */ 41119c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew TrickJEMALLOC_INLINE size_t 4120e122d1c2422285c872f68fc0ae1f7e5d2739572Andrew Trickpow2_ceil(size_t x) 41319c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick{ 414ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel 415ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel x--; 416ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel x |= x >> 1; 41736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines x |= x >> 2; 41819c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick x |= x >> 4; 4199dfa1671fc5ba478530ba88877dd508dac20bac4Devang Patel x |= x >> 8; 4209dfa1671fc5ba478530ba88877dd508dac20bac4Devang Patel x |= x >> 16; 421ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel#if (LG_SIZEOF_PTR == 3) 42240b6fdb81e12b40dd41c9f9f07befb60ec7291c3Owen Anderson x |= x >> 32; 42340b6fdb81e12b40dd41c9f9f07befb60ec7291c3Owen Anderson#endif 42440b6fdb81e12b40dd41c9f9f07befb60ec7291c3Owen Anderson x++; 42540b6fdb81e12b40dd41c9f9f07befb60ec7291c3Owen Anderson return (x); 42640b6fdb81e12b40dd41c9f9f07befb60ec7291c3Owen Anderson} 427ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel 428ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel/* 42936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines * Compute usable size that would result from allocating an object with the 43019c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick * specified size. 43140b6fdb81e12b40dd41c9f9f07befb60ec7291c3Owen Anderson */ 43240b6fdb81e12b40dd41c9f9f07befb60ec7291c3Owen AndersonJEMALLOC_INLINE size_t 43340b6fdb81e12b40dd41c9f9f07befb60ec7291c3Owen Andersons2u(size_t size) 43440b6fdb81e12b40dd41c9f9f07befb60ec7291c3Owen Anderson{ 43540b6fdb81e12b40dd41c9f9f07befb60ec7291c3Owen Anderson 436209ee187c92db27e7b38bef26a14f3b257bc2744Dan Gohman if (size <= small_maxclass) 437ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel return (arena_bin_info[SMALL_SIZE2BIN(size)].reg_size); 43836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (size <= arena_maxclass) 439ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel return (PAGE_CEILING(size)); 44036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return (CHUNK_CEILING(size)); 44136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 4423660ecabbb85b31308f38938ce3f56f0a330a84bChris Lattner 443ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel/* 444dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines * Compute usable size that would result from allocating an object with the 445ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel * specified size and alignment. 446ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel */ 447ab7752c1496c2913793305ba4b989a551c5617e1Devang PatelJEMALLOC_INLINE size_t 448ab7752c1496c2913793305ba4b989a551c5617e1Devang Patelsa2u(size_t size, size_t alignment, size_t *run_size_p) 44936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines{ 450ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel size_t usize; 45136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 452305d2533f6ef36a056d5777bdfc74077507b20b4Devang Patel /* 453305d2533f6ef36a056d5777bdfc74077507b20b4Devang Patel * Round size up to the nearest multiple of alignment. 454305d2533f6ef36a056d5777bdfc74077507b20b4Devang Patel * 455ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel * This done, we can take advantage of the fact that for each small 456310fa65ab9acb96319d0fcb779cdd35350f5d00fEvan Cheng * size class, every object is aligned at the smallest power of two 457ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel * that is non-zero in the base two representation of the size. For 458ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel * example: 459ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel * 460ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel * Size | Base 2 | Minimum alignment 46136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines * -----+----------+------------------ 46219c51a9b3e544fc3bd34488f0c9c0c8a3df73a58Andrew Trick * 96 | 1100000 | 32 463ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel * 144 | 10100000 | 32 464ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel * 192 | 11000000 | 64 465ab7752c1496c2913793305ba4b989a551c5617e1Devang Patel * 466a782e75d487006cafffdc256b3c623307fee4dcfChris Lattner * Depending on runtime settings, it is possible that arena_malloc() 4675ae563aaf9992038db63b69d16e768b9f2fcd30bDevang Patel * will further round up to a power of two, but that never causes 4687db949df789383acce98ef072f08794fdd5bd04eDan Gohman * correctness issues. 4697db949df789383acce98ef072f08794fdd5bd04eDan Gohman */ 4705ae563aaf9992038db63b69d16e768b9f2fcd30bDevang Patel usize = (size + (alignment - 1)) & (-alignment); 471 /* 472 * (usize < size) protects against the combination of maximal 473 * alignment and size greater than maximal alignment. 474 */ 475 if (usize < size) { 476 /* size_t overflow. */ 477 return (0); 478 } 479 480 if (usize <= arena_maxclass && alignment <= PAGE_SIZE) { 481 if (usize <= small_maxclass) 482 return (arena_bin_info[SMALL_SIZE2BIN(usize)].reg_size); 483 return (PAGE_CEILING(usize)); 484 } else { 485 size_t run_size; 486 487 /* 488 * We can't achieve subpage alignment, so round up alignment 489 * permanently; it makes later calculations simpler. 490 */ 491 alignment = PAGE_CEILING(alignment); 492 usize = PAGE_CEILING(size); 493 /* 494 * (usize < size) protects against very large sizes within 495 * PAGE_SIZE of SIZE_T_MAX. 496 * 497 * (usize + alignment < usize) protects against the 498 * combination of maximal alignment and usize large enough 499 * to cause overflow. This is similar to the first overflow 500 * check above, but it needs to be repeated due to the new 501 * usize value, which may now be *equal* to maximal 502 * alignment, whereas before we only detected overflow if the 503 * original size was *greater* than maximal alignment. 504 */ 505 if (usize < size || usize + alignment < usize) { 506 /* size_t overflow. */ 507 return (0); 508 } 509 510 /* 511 * Calculate the size of the over-size run that arena_palloc() 512 * would need to allocate in order to guarantee the alignment. 513 */ 514 if (usize >= alignment) 515 run_size = usize + alignment - PAGE_SIZE; 516 else { 517 /* 518 * It is possible that (alignment << 1) will cause 519 * overflow, but it doesn't matter because we also 520 * subtract PAGE_SIZE, which in the case of overflow 521 * leaves us with a very large run_size. That causes 522 * the first conditional below to fail, which means 523 * that the bogus run_size value never gets used for 524 * anything important. 525 */ 526 run_size = (alignment << 1) - PAGE_SIZE; 527 } 528 if (run_size_p != NULL) 529 *run_size_p = run_size; 530 531 if (run_size <= arena_maxclass) 532 return (PAGE_CEILING(usize)); 533 return (CHUNK_CEILING(usize)); 534 } 535} 536 537/* 538 * Wrapper around malloc_message() that avoids the need for 539 * JEMALLOC_P(malloc_message)(...) throughout the code. 540 */ 541JEMALLOC_INLINE void 542malloc_write(const char *s) 543{ 544 545 JEMALLOC_P(malloc_message)(NULL, s); 546} 547 548/* 549 * Choose an arena based on a per-thread value (fast-path code, calls slow-path 550 * code if necessary). 551 */ 552JEMALLOC_INLINE arena_t * 553choose_arena(void) 554{ 555 arena_t *ret; 556 557 ret = ARENA_GET(); 558 if (ret == NULL) { 559 ret = choose_arena_hard(); 560 assert(ret != NULL); 561 } 562 563 return (ret); 564} 565 566#if (defined(JEMALLOC_STATS) && defined(NO_TLS)) 567JEMALLOC_INLINE thread_allocated_t * 568thread_allocated_get(void) 569{ 570 thread_allocated_t *thread_allocated = (thread_allocated_t *) 571 pthread_getspecific(thread_allocated_tsd); 572 573 if (thread_allocated == NULL) 574 return (thread_allocated_get_hard()); 575 return (thread_allocated); 576} 577#endif 578#endif 579 580#include "jemalloc/internal/bitmap.h" 581#include "jemalloc/internal/rtree.h" 582#include "jemalloc/internal/tcache.h" 583#include "jemalloc/internal/arena.h" 584#include "jemalloc/internal/hash.h" 585#ifdef JEMALLOC_ZONE 586#include "jemalloc/internal/zone.h" 587#endif 588 589#ifndef JEMALLOC_ENABLE_INLINE 590void *imalloc(size_t size); 591void *icalloc(size_t size); 592void *ipalloc(size_t usize, size_t alignment, bool zero); 593size_t isalloc(const void *ptr); 594# ifdef JEMALLOC_IVSALLOC 595size_t ivsalloc(const void *ptr); 596# endif 597void idalloc(void *ptr); 598void *iralloc(void *ptr, size_t size, size_t extra, size_t alignment, 599 bool zero, bool no_move); 600#endif 601 602#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) 603JEMALLOC_INLINE void * 604imalloc(size_t size) 605{ 606 607 assert(size != 0); 608 609 if (size <= arena_maxclass) 610 return (arena_malloc(size, false)); 611 else 612 return (huge_malloc(size, false)); 613} 614 615JEMALLOC_INLINE void * 616icalloc(size_t size) 617{ 618 619 if (size <= arena_maxclass) 620 return (arena_malloc(size, true)); 621 else 622 return (huge_malloc(size, true)); 623} 624 625JEMALLOC_INLINE void * 626ipalloc(size_t usize, size_t alignment, bool zero) 627{ 628 void *ret; 629 630 assert(usize != 0); 631 assert(usize == sa2u(usize, alignment, NULL)); 632 633 if (usize <= arena_maxclass && alignment <= PAGE_SIZE) 634 ret = arena_malloc(usize, zero); 635 else { 636 size_t run_size 637#ifdef JEMALLOC_CC_SILENCE 638 = 0 639#endif 640 ; 641 642 /* 643 * Ideally we would only ever call sa2u() once per aligned 644 * allocation request, and the caller of this function has 645 * already done so once. However, it's rather burdensome to 646 * require every caller to pass in run_size, especially given 647 * that it's only relevant to large allocations. Therefore, 648 * just call it again here in order to get run_size. 649 */ 650 sa2u(usize, alignment, &run_size); 651 if (run_size <= arena_maxclass) { 652 ret = arena_palloc(choose_arena(), usize, run_size, 653 alignment, zero); 654 } else if (alignment <= chunksize) 655 ret = huge_malloc(usize, zero); 656 else 657 ret = huge_palloc(usize, alignment, zero); 658 } 659 660 assert(((uintptr_t)ret & (alignment - 1)) == 0); 661 return (ret); 662} 663 664JEMALLOC_INLINE size_t 665isalloc(const void *ptr) 666{ 667 size_t ret; 668 arena_chunk_t *chunk; 669 670 assert(ptr != NULL); 671 672 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 673 if (chunk != ptr) { 674 /* Region. */ 675 dassert(chunk->arena->magic == ARENA_MAGIC); 676 677#ifdef JEMALLOC_PROF 678 ret = arena_salloc_demote(ptr); 679#else 680 ret = arena_salloc(ptr); 681#endif 682 } else 683 ret = huge_salloc(ptr); 684 685 return (ret); 686} 687 688#ifdef JEMALLOC_IVSALLOC 689JEMALLOC_INLINE size_t 690ivsalloc(const void *ptr) 691{ 692 693 /* Return 0 if ptr is not within a chunk managed by jemalloc. */ 694 if (rtree_get(chunks_rtree, (uintptr_t)CHUNK_ADDR2BASE(ptr)) == NULL) 695 return (0); 696 697 return (isalloc(ptr)); 698} 699#endif 700 701JEMALLOC_INLINE void 702idalloc(void *ptr) 703{ 704 arena_chunk_t *chunk; 705 706 assert(ptr != NULL); 707 708 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 709 if (chunk != ptr) 710 arena_dalloc(chunk->arena, chunk, ptr); 711 else 712 huge_dalloc(ptr, true); 713} 714 715JEMALLOC_INLINE void * 716iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, 717 bool no_move) 718{ 719 void *ret; 720 size_t oldsize; 721 722 assert(ptr != NULL); 723 assert(size != 0); 724 725 oldsize = isalloc(ptr); 726 727 if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) 728 != 0) { 729 size_t usize, copysize; 730 731 /* 732 * Existing object alignment is inadquate; allocate new space 733 * and copy. 734 */ 735 if (no_move) 736 return (NULL); 737 usize = sa2u(size + extra, alignment, NULL); 738 if (usize == 0) 739 return (NULL); 740 ret = ipalloc(usize, alignment, zero); 741 if (ret == NULL) { 742 if (extra == 0) 743 return (NULL); 744 /* Try again, without extra this time. */ 745 usize = sa2u(size, alignment, NULL); 746 if (usize == 0) 747 return (NULL); 748 ret = ipalloc(usize, alignment, zero); 749 if (ret == NULL) 750 return (NULL); 751 } 752 /* 753 * Copy at most size bytes (not size+extra), since the caller 754 * has no expectation that the extra bytes will be reliably 755 * preserved. 756 */ 757 copysize = (size < oldsize) ? size : oldsize; 758 memcpy(ret, ptr, copysize); 759 idalloc(ptr); 760 return (ret); 761 } 762 763 if (no_move) { 764 if (size <= arena_maxclass) { 765 return (arena_ralloc_no_move(ptr, oldsize, size, 766 extra, zero)); 767 } else { 768 return (huge_ralloc_no_move(ptr, oldsize, size, 769 extra)); 770 } 771 } else { 772 if (size + extra <= arena_maxclass) { 773 return (arena_ralloc(ptr, oldsize, size, extra, 774 alignment, zero)); 775 } else { 776 return (huge_ralloc(ptr, oldsize, size, extra, 777 alignment, zero)); 778 } 779 } 780} 781#endif 782 783#include "jemalloc/internal/prof.h" 784 785#undef JEMALLOC_H_INLINES 786/******************************************************************************/ 787