1/******************************************************************************/ 2#ifdef JEMALLOC_H_TYPES 3 4#ifdef _WIN32 5# ifdef _WIN64 6# define FMT64_PREFIX "ll" 7# define FMTPTR_PREFIX "ll" 8# else 9# define FMT64_PREFIX "ll" 10# define FMTPTR_PREFIX "" 11# endif 12# define FMTd32 "d" 13# define FMTu32 "u" 14# define FMTx32 "x" 15# define FMTd64 FMT64_PREFIX "d" 16# define FMTu64 FMT64_PREFIX "u" 17# define FMTx64 FMT64_PREFIX "x" 18# define FMTdPTR FMTPTR_PREFIX "d" 19# define FMTuPTR FMTPTR_PREFIX "u" 20# define FMTxPTR FMTPTR_PREFIX "x" 21#else 22# include <inttypes.h> 23# define FMTd32 PRId32 24# define FMTu32 PRIu32 25# define FMTx32 PRIx32 26# define FMTd64 PRId64 27# define FMTu64 PRIu64 28# define FMTx64 PRIx64 29# define FMTdPTR PRIdPTR 30# define FMTuPTR PRIuPTR 31# define FMTxPTR PRIxPTR 32#endif 33 34/* Size of stack-allocated buffer passed to buferror(). */ 35#define BUFERROR_BUF 64 36 37/* 38 * Size of stack-allocated buffer used by malloc_{,v,vc}printf(). This must be 39 * large enough for all possible uses within jemalloc. 40 */ 41#define MALLOC_PRINTF_BUFSIZE 4096 42 43/* 44 * Wrap a cpp argument that contains commas such that it isn't broken up into 45 * multiple arguments. 46 */ 47#define JEMALLOC_ARG_CONCAT(...) __VA_ARGS__ 48 49/* 50 * Silence compiler warnings due to uninitialized values. This is used 51 * wherever the compiler fails to recognize that the variable is never used 52 * uninitialized. 53 */ 54#ifdef JEMALLOC_CC_SILENCE 55# define JEMALLOC_CC_SILENCE_INIT(v) = v 56#else 57# define JEMALLOC_CC_SILENCE_INIT(v) 58#endif 59 60#define JEMALLOC_GNUC_PREREQ(major, minor) \ 61 (!defined(__clang__) && \ 62 (__GNUC__ > (major) || (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))) 63#ifndef __has_builtin 64# define __has_builtin(builtin) (0) 65#endif 66#define JEMALLOC_CLANG_HAS_BUILTIN(builtin) \ 67 (defined(__clang__) && __has_builtin(builtin)) 68 69#ifdef __GNUC__ 70# define likely(x) __builtin_expect(!!(x), 1) 71# define unlikely(x) __builtin_expect(!!(x), 0) 72# if JEMALLOC_GNUC_PREREQ(4, 6) || \ 73 JEMALLOC_CLANG_HAS_BUILTIN(__builtin_unreachable) 74# define unreachable() __builtin_unreachable() 75# else 76# define unreachable() 77# endif 78#else 79# define likely(x) !!(x) 80# define unlikely(x) !!(x) 81# define unreachable() 82#endif 83 84#include "jemalloc/internal/assert.h" 85 86/* Use to assert a particular configuration, e.g., cassert(config_debug). */ 87#define cassert(c) do { \ 88 if (unlikely(!(c))) \ 89 not_reached(); \ 90} while (0) 91 92#endif /* JEMALLOC_H_TYPES */ 93/******************************************************************************/ 94#ifdef JEMALLOC_H_STRUCTS 95 96#endif /* JEMALLOC_H_STRUCTS */ 97/******************************************************************************/ 98#ifdef JEMALLOC_H_EXTERNS 99 100int buferror(int err, char *buf, size_t buflen); 101uintmax_t malloc_strtoumax(const char *restrict nptr, 102 char **restrict endptr, int base); 103void malloc_write(const char *s); 104 105/* 106 * malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating 107 * point math. 108 */ 109int malloc_vsnprintf(char *str, size_t size, const char *format, 110 va_list ap); 111int malloc_snprintf(char *str, size_t size, const char *format, ...) 112 JEMALLOC_FORMAT_PRINTF(3, 4); 113void malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque, 114 const char *format, va_list ap); 115void malloc_cprintf(void (*write)(void *, const char *), void *cbopaque, 116 const char *format, ...) JEMALLOC_FORMAT_PRINTF(3, 4); 117void malloc_printf(const char *format, ...) JEMALLOC_FORMAT_PRINTF(1, 2); 118 119#endif /* JEMALLOC_H_EXTERNS */ 120/******************************************************************************/ 121#ifdef JEMALLOC_H_INLINES 122 123#ifndef JEMALLOC_ENABLE_INLINE 124unsigned ffs_llu(unsigned long long bitmap); 125unsigned ffs_lu(unsigned long bitmap); 126unsigned ffs_u(unsigned bitmap); 127unsigned ffs_zu(size_t bitmap); 128unsigned ffs_u64(uint64_t bitmap); 129unsigned ffs_u32(uint32_t bitmap); 130uint64_t pow2_ceil_u64(uint64_t x); 131uint32_t pow2_ceil_u32(uint32_t x); 132size_t pow2_ceil_zu(size_t x); 133unsigned lg_floor(size_t x); 134void set_errno(int errnum); 135int get_errno(void); 136#endif 137 138#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_UTIL_C_)) 139 140/* Sanity check. */ 141#if !defined(JEMALLOC_INTERNAL_FFSLL) || !defined(JEMALLOC_INTERNAL_FFSL) \ 142 || !defined(JEMALLOC_INTERNAL_FFS) 143# error JEMALLOC_INTERNAL_FFS{,L,LL} should have been defined by configure 144#endif 145 146JEMALLOC_ALWAYS_INLINE unsigned 147ffs_llu(unsigned long long bitmap) 148{ 149 150 return (JEMALLOC_INTERNAL_FFSLL(bitmap)); 151} 152 153JEMALLOC_ALWAYS_INLINE unsigned 154ffs_lu(unsigned long bitmap) 155{ 156 157 return (JEMALLOC_INTERNAL_FFSL(bitmap)); 158} 159 160JEMALLOC_ALWAYS_INLINE unsigned 161ffs_u(unsigned bitmap) 162{ 163 164 return (JEMALLOC_INTERNAL_FFS(bitmap)); 165} 166 167JEMALLOC_ALWAYS_INLINE unsigned 168ffs_zu(size_t bitmap) 169{ 170 171#if LG_SIZEOF_PTR == LG_SIZEOF_INT 172 return (ffs_u(bitmap)); 173#elif LG_SIZEOF_PTR == LG_SIZEOF_LONG 174 return (ffs_lu(bitmap)); 175#elif LG_SIZEOF_PTR == LG_SIZEOF_LONG_LONG 176 return (ffs_llu(bitmap)); 177#else 178#error No implementation for size_t ffs() 179#endif 180} 181 182JEMALLOC_ALWAYS_INLINE unsigned 183ffs_u64(uint64_t bitmap) 184{ 185 186#if LG_SIZEOF_LONG == 3 187 return (ffs_lu(bitmap)); 188#elif LG_SIZEOF_LONG_LONG == 3 189 return (ffs_llu(bitmap)); 190#else 191#error No implementation for 64-bit ffs() 192#endif 193} 194 195JEMALLOC_ALWAYS_INLINE unsigned 196ffs_u32(uint32_t bitmap) 197{ 198 199#if LG_SIZEOF_INT == 2 200 return (ffs_u(bitmap)); 201#else 202#error No implementation for 32-bit ffs() 203#endif 204 return (ffs_u(bitmap)); 205} 206 207JEMALLOC_INLINE uint64_t 208pow2_ceil_u64(uint64_t x) 209{ 210 211 x--; 212 x |= x >> 1; 213 x |= x >> 2; 214 x |= x >> 4; 215 x |= x >> 8; 216 x |= x >> 16; 217 x |= x >> 32; 218 x++; 219 return (x); 220} 221 222JEMALLOC_INLINE uint32_t 223pow2_ceil_u32(uint32_t x) 224{ 225 226 x--; 227 x |= x >> 1; 228 x |= x >> 2; 229 x |= x >> 4; 230 x |= x >> 8; 231 x |= x >> 16; 232 x++; 233 return (x); 234} 235 236/* Compute the smallest power of 2 that is >= x. */ 237JEMALLOC_INLINE size_t 238pow2_ceil_zu(size_t x) 239{ 240 241#if (LG_SIZEOF_PTR == 3) 242 return (pow2_ceil_u64(x)); 243#else 244 return (pow2_ceil_u32(x)); 245#endif 246} 247 248#if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__)) 249JEMALLOC_INLINE unsigned 250lg_floor(size_t x) 251{ 252 size_t ret; 253 254 assert(x != 0); 255 256 asm ("bsr %1, %0" 257 : "=r"(ret) // Outputs. 258 : "r"(x) // Inputs. 259 ); 260 assert(ret < UINT_MAX); 261 return ((unsigned)ret); 262} 263#elif (defined(_MSC_VER)) 264JEMALLOC_INLINE unsigned 265lg_floor(size_t x) 266{ 267 unsigned long ret; 268 269 assert(x != 0); 270 271#if (LG_SIZEOF_PTR == 3) 272 _BitScanReverse64(&ret, x); 273#elif (LG_SIZEOF_PTR == 2) 274 _BitScanReverse(&ret, x); 275#else 276# error "Unsupported type size for lg_floor()" 277#endif 278 assert(ret < UINT_MAX); 279 return ((unsigned)ret); 280} 281#elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ)) 282JEMALLOC_INLINE unsigned 283lg_floor(size_t x) 284{ 285 286 assert(x != 0); 287 288#if (LG_SIZEOF_PTR == LG_SIZEOF_INT) 289 return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clz(x)); 290#elif (LG_SIZEOF_PTR == LG_SIZEOF_LONG) 291 return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clzl(x)); 292#else 293# error "Unsupported type size for lg_floor()" 294#endif 295} 296#else 297JEMALLOC_INLINE unsigned 298lg_floor(size_t x) 299{ 300 301 assert(x != 0); 302 303 x |= (x >> 1); 304 x |= (x >> 2); 305 x |= (x >> 4); 306 x |= (x >> 8); 307 x |= (x >> 16); 308#if (LG_SIZEOF_PTR == 3) 309 x |= (x >> 32); 310#endif 311 if (x == SIZE_T_MAX) 312 return ((8 << LG_SIZEOF_PTR) - 1); 313 x++; 314 return (ffs_zu(x) - 2); 315} 316#endif 317 318/* Set error code. */ 319JEMALLOC_INLINE void 320set_errno(int errnum) 321{ 322 323#ifdef _WIN32 324 SetLastError(errnum); 325#else 326 errno = errnum; 327#endif 328} 329 330/* Get last error code. */ 331JEMALLOC_INLINE int 332get_errno(void) 333{ 334 335#ifdef _WIN32 336 return (GetLastError()); 337#else 338 return (errno); 339#endif 340} 341#endif 342 343#endif /* JEMALLOC_H_INLINES */ 344/******************************************************************************/ 345