1d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans/******************************************************************************/ 2d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans#ifdef JEMALLOC_H_TYPES 3d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans 45fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans#ifdef _WIN32 55fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans# ifdef _WIN64 65fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans# define FMT64_PREFIX "ll" 75fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans# define FMTPTR_PREFIX "ll" 85fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans# else 95fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans# define FMT64_PREFIX "ll" 105fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans# define FMTPTR_PREFIX "" 115fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans# endif 125fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans# define FMTd32 "d" 135fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans# define FMTu32 "u" 145fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans# define FMTx32 "x" 155fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans# define FMTd64 FMT64_PREFIX "d" 165fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans# define FMTu64 FMT64_PREFIX "u" 175fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans# define FMTx64 FMT64_PREFIX "x" 185fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans# define FMTdPTR FMTPTR_PREFIX "d" 195fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans# define FMTuPTR FMTPTR_PREFIX "u" 205fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans# define FMTxPTR FMTPTR_PREFIX "x" 215fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans#else 225fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans# include <inttypes.h> 235fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans# define FMTd32 PRId32 245fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans# define FMTu32 PRIu32 255fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans# define FMTx32 PRIx32 265fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans# define FMTd64 PRId64 275fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans# define FMTu64 PRIu64 285fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans# define FMTx64 PRIx64 295fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans# define FMTdPTR PRIdPTR 305fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans# define FMTuPTR PRIuPTR 315fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans# define FMTxPTR PRIxPTR 325fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans#endif 335fae7dc1b316d0e93aa20cc3aaf050f509aec705Jason Evans 34d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans/* Size of stack-allocated buffer passed to buferror(). */ 35d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans#define BUFERROR_BUF 64 36d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans 37d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans/* 38cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans * Size of stack-allocated buffer used by malloc_{,v,vc}printf(). This must be 39cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans * large enough for all possible uses within jemalloc. 40824d34e5b7f5cf00bf472ec79f7ec1c6e3474114Jason Evans */ 41824d34e5b7f5cf00bf472ec79f7ec1c6e3474114Jason Evans#define MALLOC_PRINTF_BUFSIZE 4096 42824d34e5b7f5cf00bf472ec79f7ec1c6e3474114Jason Evans 43a82070ef5fc3aa81fda43086cdcc22bfa826b894Chris Peterson/* Junk fill patterns. */ 44fc11f3cb8443c029f54bf9ba21574b0f61996dd2Jason Evans#ifndef JEMALLOC_ALLOC_JUNK 45fc11f3cb8443c029f54bf9ba21574b0f61996dd2Jason Evans# define JEMALLOC_ALLOC_JUNK ((uint8_t)0xa5) 46fc11f3cb8443c029f54bf9ba21574b0f61996dd2Jason Evans#endif 47fc11f3cb8443c029f54bf9ba21574b0f61996dd2Jason Evans#ifndef JEMALLOC_FREE_JUNK 48fc11f3cb8443c029f54bf9ba21574b0f61996dd2Jason Evans# define JEMALLOC_FREE_JUNK ((uint8_t)0x5a) 49fc11f3cb8443c029f54bf9ba21574b0f61996dd2Jason Evans#endif 50a82070ef5fc3aa81fda43086cdcc22bfa826b894Chris Peterson 51824d34e5b7f5cf00bf472ec79f7ec1c6e3474114Jason Evans/* 52cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans * Wrap a cpp argument that contains commas such that it isn't broken up into 53cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans * multiple arguments. 54cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans */ 55a4f124f59fa5f702231432a7e5fa45140ba81e2aJason Evans#define JEMALLOC_ARG_CONCAT(...) __VA_ARGS__ 56cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans 57cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans/* 589225a1991a58190207cca2ff3cdba966bb322dd5Jason Evans * Silence compiler warnings due to uninitialized values. This is used 599225a1991a58190207cca2ff3cdba966bb322dd5Jason Evans * wherever the compiler fails to recognize that the variable is never used 609225a1991a58190207cca2ff3cdba966bb322dd5Jason Evans * uninitialized. 619225a1991a58190207cca2ff3cdba966bb322dd5Jason Evans */ 629225a1991a58190207cca2ff3cdba966bb322dd5Jason Evans#ifdef JEMALLOC_CC_SILENCE 638ad6bf360f9ca5c6c9a1d8e5825ee473bb4697daIgor Podlesny# define JEMALLOC_CC_SILENCE_INIT(v) = v 649225a1991a58190207cca2ff3cdba966bb322dd5Jason Evans#else 658ad6bf360f9ca5c6c9a1d8e5825ee473bb4697daIgor Podlesny# define JEMALLOC_CC_SILENCE_INIT(v) 669225a1991a58190207cca2ff3cdba966bb322dd5Jason Evans#endif 679225a1991a58190207cca2ff3cdba966bb322dd5Jason Evans 686b5609d23bf49423fdc6506281e0deac7c3a524eDaniel Micay#ifdef __GNUC__ 698ad6bf360f9ca5c6c9a1d8e5825ee473bb4697daIgor Podlesny# define likely(x) __builtin_expect(!!(x), 1) 708ad6bf360f9ca5c6c9a1d8e5825ee473bb4697daIgor Podlesny# define unlikely(x) __builtin_expect(!!(x), 0) 716b5609d23bf49423fdc6506281e0deac7c3a524eDaniel Micay#else 728ad6bf360f9ca5c6c9a1d8e5825ee473bb4697daIgor Podlesny# define likely(x) !!(x) 738ad6bf360f9ca5c6c9a1d8e5825ee473bb4697daIgor Podlesny# define unlikely(x) !!(x) 746b5609d23bf49423fdc6506281e0deac7c3a524eDaniel Micay#endif 756b5609d23bf49423fdc6506281e0deac7c3a524eDaniel Micay 76d1207f0d371eade218f7572743b5eddedc7fff94Elliot Ronaghan#if !defined(JEMALLOC_INTERNAL_UNREACHABLE) 77d1207f0d371eade218f7572743b5eddedc7fff94Elliot Ronaghan# error JEMALLOC_INTERNAL_UNREACHABLE should have been defined by configure 78d1207f0d371eade218f7572743b5eddedc7fff94Elliot Ronaghan#endif 79d1207f0d371eade218f7572743b5eddedc7fff94Elliot Ronaghan 80d1207f0d371eade218f7572743b5eddedc7fff94Elliot Ronaghan#define unreachable() JEMALLOC_INTERNAL_UNREACHABLE() 81d1207f0d371eade218f7572743b5eddedc7fff94Elliot Ronaghan 82f9e3459f751b08b3c2108fda7462827cf8a4f2afJason Evans#include "jemalloc/internal/assert.h" 83d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans 846556e28be15d9acd8f3835fb9fad90145e1edbffJason Evans/* Use to assert a particular configuration, e.g., cassert(config_debug). */ 856556e28be15d9acd8f3835fb9fad90145e1edbffJason Evans#define cassert(c) do { \ 8623fdf8b359a690f457c5300338f4994d06402b95Daniel Micay if (unlikely(!(c))) \ 876556e28be15d9acd8f3835fb9fad90145e1edbffJason Evans not_reached(); \ 886556e28be15d9acd8f3835fb9fad90145e1edbffJason Evans} while (0) 896556e28be15d9acd8f3835fb9fad90145e1edbffJason Evans 90d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans#endif /* JEMALLOC_H_TYPES */ 91d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans/******************************************************************************/ 92d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans#ifdef JEMALLOC_H_STRUCTS 93d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans 94d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans#endif /* JEMALLOC_H_STRUCTS */ 95d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans/******************************************************************************/ 96d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans#ifdef JEMALLOC_H_EXTERNS 97d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans 982a83ed0284e92c7ba4bd4efe9df149ac724b2f26Jason Evansint buferror(int err, char *buf, size_t buflen); 99e18c25d23de0e845f0ee7e11d02c1be044738a3cJason Evansuintmax_t malloc_strtoumax(const char *restrict nptr, 100e18c25d23de0e845f0ee7e11d02c1be044738a3cJason Evans char **restrict endptr, int base); 101889ec59bd3ae3190fb715e64d8d15b6a1b47314aJason Evansvoid malloc_write(const char *s); 102d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans 103d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans/* 104d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans * malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating 105d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans * point math. 106d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans */ 10722af74e10615ce6b6898ae38a378af27757f9e16Jason Evanssize_t malloc_vsnprintf(char *str, size_t size, const char *format, 108d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans va_list ap); 10922af74e10615ce6b6898ae38a378af27757f9e16Jason Evanssize_t malloc_snprintf(char *str, size_t size, const char *format, ...) 110e42c309eba6c5084dc0abda9b211e91e2c548fdfJason Evans JEMALLOC_FORMAT_PRINTF(3, 4); 111d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evansvoid malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque, 112d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans const char *format, va_list ap); 113d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evansvoid malloc_cprintf(void (*write)(void *, const char *), void *cbopaque, 114e42c309eba6c5084dc0abda9b211e91e2c548fdfJason Evans const char *format, ...) JEMALLOC_FORMAT_PRINTF(3, 4); 115e42c309eba6c5084dc0abda9b211e91e2c548fdfJason Evansvoid malloc_printf(const char *format, ...) JEMALLOC_FORMAT_PRINTF(1, 2); 116d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans 117d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans#endif /* JEMALLOC_H_EXTERNS */ 118d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans/******************************************************************************/ 119d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans#ifdef JEMALLOC_H_INLINES 120d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans 121d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans#ifndef JEMALLOC_ENABLE_INLINE 1229f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evansunsigned ffs_llu(unsigned long long bitmap); 1239f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evansunsigned ffs_lu(unsigned long bitmap); 1249f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evansunsigned ffs_u(unsigned bitmap); 1259f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evansunsigned ffs_zu(size_t bitmap); 1269f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evansunsigned ffs_u64(uint64_t bitmap); 1279f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evansunsigned ffs_u32(uint32_t bitmap); 12834676d33690f6cc6885ff769e537ca940aacf886Jason Evansuint64_t pow2_ceil_u64(uint64_t x); 12934676d33690f6cc6885ff769e537ca940aacf886Jason Evansuint32_t pow2_ceil_u32(uint32_t x); 13034676d33690f6cc6885ff769e537ca940aacf886Jason Evanssize_t pow2_ceil_zu(size_t x); 1311c42a04cc6d3cc5d92bec55432015785584a4b0dJason Evansunsigned lg_floor(size_t x); 132a14bce85e885f83c96116cc5438ae52d740f3727Mike Hommeyvoid set_errno(int errnum); 133a14bce85e885f83c96116cc5438ae52d740f3727Mike Hommeyint get_errno(void); 134d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans#endif 135d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans 136d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_UTIL_C_)) 1379c3a10fdf6baa5ddb042b6adbef1ff1b3c613ce3Richard Diamond 138e12eaf93dca308a426c182956197b0eeb5f2cff3Jason Evans/* Sanity check. */ 139ecae12323d44cd739662051a2b9a5965cbe0e965Jason Evans#if !defined(JEMALLOC_INTERNAL_FFSLL) || !defined(JEMALLOC_INTERNAL_FFSL) \ 140ecae12323d44cd739662051a2b9a5965cbe0e965Jason Evans || !defined(JEMALLOC_INTERNAL_FFS) 141ecae12323d44cd739662051a2b9a5965cbe0e965Jason Evans# error JEMALLOC_INTERNAL_FFS{,L,LL} should have been defined by configure 1429c3a10fdf6baa5ddb042b6adbef1ff1b3c613ce3Richard Diamond#endif 1439c3a10fdf6baa5ddb042b6adbef1ff1b3c613ce3Richard Diamond 1449f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason EvansJEMALLOC_ALWAYS_INLINE unsigned 1459f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evansffs_llu(unsigned long long bitmap) 146ecae12323d44cd739662051a2b9a5965cbe0e965Jason Evans{ 147ecae12323d44cd739662051a2b9a5965cbe0e965Jason Evans 148ecae12323d44cd739662051a2b9a5965cbe0e965Jason Evans return (JEMALLOC_INTERNAL_FFSLL(bitmap)); 149ecae12323d44cd739662051a2b9a5965cbe0e965Jason Evans} 150ecae12323d44cd739662051a2b9a5965cbe0e965Jason Evans 1519f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason EvansJEMALLOC_ALWAYS_INLINE unsigned 1529f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evansffs_lu(unsigned long bitmap) 1539c3a10fdf6baa5ddb042b6adbef1ff1b3c613ce3Richard Diamond{ 1549c3a10fdf6baa5ddb042b6adbef1ff1b3c613ce3Richard Diamond 155c810fcea1fa7983ef5bcabe6556cdc19dde6dd8dJason Evans return (JEMALLOC_INTERNAL_FFSL(bitmap)); 1569c3a10fdf6baa5ddb042b6adbef1ff1b3c613ce3Richard Diamond} 1579c3a10fdf6baa5ddb042b6adbef1ff1b3c613ce3Richard Diamond 1589f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason EvansJEMALLOC_ALWAYS_INLINE unsigned 1599f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evansffs_u(unsigned bitmap) 1609c3a10fdf6baa5ddb042b6adbef1ff1b3c613ce3Richard Diamond{ 1619c3a10fdf6baa5ddb042b6adbef1ff1b3c613ce3Richard Diamond 162c810fcea1fa7983ef5bcabe6556cdc19dde6dd8dJason Evans return (JEMALLOC_INTERNAL_FFS(bitmap)); 1639c3a10fdf6baa5ddb042b6adbef1ff1b3c613ce3Richard Diamond} 1649c3a10fdf6baa5ddb042b6adbef1ff1b3c613ce3Richard Diamond 1659f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason EvansJEMALLOC_ALWAYS_INLINE unsigned 1669f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evansffs_zu(size_t bitmap) 1679f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans{ 1689f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans 169aa63d5d377b4508b83502e923690d1d7b67c8c88Jason Evans#if LG_SIZEOF_PTR == LG_SIZEOF_INT 1709f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans return (ffs_u(bitmap)); 171aa63d5d377b4508b83502e923690d1d7b67c8c88Jason Evans#elif LG_SIZEOF_PTR == LG_SIZEOF_LONG 172aa63d5d377b4508b83502e923690d1d7b67c8c88Jason Evans return (ffs_lu(bitmap)); 173aa63d5d377b4508b83502e923690d1d7b67c8c88Jason Evans#elif LG_SIZEOF_PTR == LG_SIZEOF_LONG_LONG 174aa63d5d377b4508b83502e923690d1d7b67c8c88Jason Evans return (ffs_llu(bitmap)); 1759f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans#else 1769f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans#error No implementation for size_t ffs() 1779f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans#endif 1789f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans} 1799f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans 1809f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason EvansJEMALLOC_ALWAYS_INLINE unsigned 1819f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evansffs_u64(uint64_t bitmap) 1829f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans{ 1839f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans 1849f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans#if LG_SIZEOF_LONG == 3 1859f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans return (ffs_lu(bitmap)); 1869f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans#elif LG_SIZEOF_LONG_LONG == 3 1879f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans return (ffs_llu(bitmap)); 1889f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans#else 1899f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans#error No implementation for 64-bit ffs() 1909f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans#endif 1919f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans} 1929f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans 1939f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason EvansJEMALLOC_ALWAYS_INLINE unsigned 1949f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evansffs_u32(uint32_t bitmap) 1959f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans{ 1969f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans 1979f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans#if LG_SIZEOF_INT == 2 1989f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans return (ffs_u(bitmap)); 1999f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans#else 2009f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans#error No implementation for 32-bit ffs() 2019f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans#endif 2029f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans return (ffs_u(bitmap)); 2039f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans} 2049f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans 20534676d33690f6cc6885ff769e537ca940aacf886Jason EvansJEMALLOC_INLINE uint64_t 20634676d33690f6cc6885ff769e537ca940aacf886Jason Evanspow2_ceil_u64(uint64_t x) 207d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans{ 208d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans 209d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans x--; 210d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans x |= x >> 1; 211d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans x |= x >> 2; 212d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans x |= x >> 4; 213d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans x |= x >> 8; 214d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans x |= x >> 16; 215d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans x |= x >> 32; 216d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans x++; 217d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans return (x); 218d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans} 219d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans 22034676d33690f6cc6885ff769e537ca940aacf886Jason EvansJEMALLOC_INLINE uint32_t 22134676d33690f6cc6885ff769e537ca940aacf886Jason Evanspow2_ceil_u32(uint32_t x) 22234676d33690f6cc6885ff769e537ca940aacf886Jason Evans{ 22334676d33690f6cc6885ff769e537ca940aacf886Jason Evans 22434676d33690f6cc6885ff769e537ca940aacf886Jason Evans x--; 22534676d33690f6cc6885ff769e537ca940aacf886Jason Evans x |= x >> 1; 22634676d33690f6cc6885ff769e537ca940aacf886Jason Evans x |= x >> 2; 22734676d33690f6cc6885ff769e537ca940aacf886Jason Evans x |= x >> 4; 22834676d33690f6cc6885ff769e537ca940aacf886Jason Evans x |= x >> 8; 22934676d33690f6cc6885ff769e537ca940aacf886Jason Evans x |= x >> 16; 23034676d33690f6cc6885ff769e537ca940aacf886Jason Evans x++; 23134676d33690f6cc6885ff769e537ca940aacf886Jason Evans return (x); 23234676d33690f6cc6885ff769e537ca940aacf886Jason Evans} 23334676d33690f6cc6885ff769e537ca940aacf886Jason Evans 23434676d33690f6cc6885ff769e537ca940aacf886Jason Evans/* Compute the smallest power of 2 that is >= x. */ 23534676d33690f6cc6885ff769e537ca940aacf886Jason EvansJEMALLOC_INLINE size_t 23634676d33690f6cc6885ff769e537ca940aacf886Jason Evanspow2_ceil_zu(size_t x) 23734676d33690f6cc6885ff769e537ca940aacf886Jason Evans{ 23834676d33690f6cc6885ff769e537ca940aacf886Jason Evans 23934676d33690f6cc6885ff769e537ca940aacf886Jason Evans#if (LG_SIZEOF_PTR == 3) 24034676d33690f6cc6885ff769e537ca940aacf886Jason Evans return (pow2_ceil_u64(x)); 24134676d33690f6cc6885ff769e537ca940aacf886Jason Evans#else 24234676d33690f6cc6885ff769e537ca940aacf886Jason Evans return (pow2_ceil_u32(x)); 24334676d33690f6cc6885ff769e537ca940aacf886Jason Evans#endif 24434676d33690f6cc6885ff769e537ca940aacf886Jason Evans} 24534676d33690f6cc6885ff769e537ca940aacf886Jason Evans 246d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans#if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__)) 2471c42a04cc6d3cc5d92bec55432015785584a4b0dJason EvansJEMALLOC_INLINE unsigned 248d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanslg_floor(size_t x) 249d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans{ 250d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t ret; 251d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 252c810fcea1fa7983ef5bcabe6556cdc19dde6dd8dJason Evans assert(x != 0); 253c810fcea1fa7983ef5bcabe6556cdc19dde6dd8dJason Evans 254d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans asm ("bsr %1, %0" 255d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans : "=r"(ret) // Outputs. 256d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans : "r"(x) // Inputs. 257d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans ); 2581c42a04cc6d3cc5d92bec55432015785584a4b0dJason Evans assert(ret < UINT_MAX); 2591c42a04cc6d3cc5d92bec55432015785584a4b0dJason Evans return ((unsigned)ret); 260d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans} 261112704cfbfacfc9cecdfb732741df47eb4133902Dave Rigby#elif (defined(_MSC_VER)) 2621c42a04cc6d3cc5d92bec55432015785584a4b0dJason EvansJEMALLOC_INLINE unsigned 263112704cfbfacfc9cecdfb732741df47eb4133902Dave Rigbylg_floor(size_t x) 264112704cfbfacfc9cecdfb732741df47eb4133902Dave Rigby{ 265c810fcea1fa7983ef5bcabe6556cdc19dde6dd8dJason Evans unsigned long ret; 266c810fcea1fa7983ef5bcabe6556cdc19dde6dd8dJason Evans 267c810fcea1fa7983ef5bcabe6556cdc19dde6dd8dJason Evans assert(x != 0); 268112704cfbfacfc9cecdfb732741df47eb4133902Dave Rigby 269112704cfbfacfc9cecdfb732741df47eb4133902Dave Rigby#if (LG_SIZEOF_PTR == 3) 270c810fcea1fa7983ef5bcabe6556cdc19dde6dd8dJason Evans _BitScanReverse64(&ret, x); 271112704cfbfacfc9cecdfb732741df47eb4133902Dave Rigby#elif (LG_SIZEOF_PTR == 2) 272c810fcea1fa7983ef5bcabe6556cdc19dde6dd8dJason Evans _BitScanReverse(&ret, x); 273112704cfbfacfc9cecdfb732741df47eb4133902Dave Rigby#else 2749f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans# error "Unsupported type size for lg_floor()" 275112704cfbfacfc9cecdfb732741df47eb4133902Dave Rigby#endif 2761c42a04cc6d3cc5d92bec55432015785584a4b0dJason Evans assert(ret < UINT_MAX); 2771c42a04cc6d3cc5d92bec55432015785584a4b0dJason Evans return ((unsigned)ret); 278112704cfbfacfc9cecdfb732741df47eb4133902Dave Rigby} 279d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans#elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ)) 2801c42a04cc6d3cc5d92bec55432015785584a4b0dJason EvansJEMALLOC_INLINE unsigned 281d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanslg_floor(size_t x) 282d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans{ 283d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 284c810fcea1fa7983ef5bcabe6556cdc19dde6dd8dJason Evans assert(x != 0); 285c810fcea1fa7983ef5bcabe6556cdc19dde6dd8dJason Evans 286d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans#if (LG_SIZEOF_PTR == LG_SIZEOF_INT) 2870b5c92213fbafc52c5b5a5dc84e91eacc812ae0bJason Evans return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clz(x)); 288d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans#elif (LG_SIZEOF_PTR == LG_SIZEOF_LONG) 2890b5c92213fbafc52c5b5a5dc84e91eacc812ae0bJason Evans return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clzl(x)); 290d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans#else 2919f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans# error "Unsupported type size for lg_floor()" 292d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans#endif 293d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans} 294d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans#else 2951c42a04cc6d3cc5d92bec55432015785584a4b0dJason EvansJEMALLOC_INLINE unsigned 296d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanslg_floor(size_t x) 297d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans{ 298d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 299c810fcea1fa7983ef5bcabe6556cdc19dde6dd8dJason Evans assert(x != 0); 300c810fcea1fa7983ef5bcabe6556cdc19dde6dd8dJason Evans 3010b5c92213fbafc52c5b5a5dc84e91eacc812ae0bJason Evans x |= (x >> 1); 3020b5c92213fbafc52c5b5a5dc84e91eacc812ae0bJason Evans x |= (x >> 2); 3030b5c92213fbafc52c5b5a5dc84e91eacc812ae0bJason Evans x |= (x >> 4); 3040b5c92213fbafc52c5b5a5dc84e91eacc812ae0bJason Evans x |= (x >> 8); 3050b5c92213fbafc52c5b5a5dc84e91eacc812ae0bJason Evans x |= (x >> 16); 3069f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans#if (LG_SIZEOF_PTR == 3) 3070b5c92213fbafc52c5b5a5dc84e91eacc812ae0bJason Evans x |= (x >> 32); 308d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans#endif 3099f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans if (x == SIZE_T_MAX) 3109f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans return ((8 << LG_SIZEOF_PTR) - 1); 3119f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans x++; 3129f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans return (ffs_zu(x) - 2); 313d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans} 314d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans#endif 315d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 316e12eaf93dca308a426c182956197b0eeb5f2cff3Jason Evans/* Set error code. */ 317a14bce85e885f83c96116cc5438ae52d740f3727Mike HommeyJEMALLOC_INLINE void 318a14bce85e885f83c96116cc5438ae52d740f3727Mike Hommeyset_errno(int errnum) 319a14bce85e885f83c96116cc5438ae52d740f3727Mike Hommey{ 320a14bce85e885f83c96116cc5438ae52d740f3727Mike Hommey 321a14bce85e885f83c96116cc5438ae52d740f3727Mike Hommey#ifdef _WIN32 322a14bce85e885f83c96116cc5438ae52d740f3727Mike Hommey SetLastError(errnum); 323a14bce85e885f83c96116cc5438ae52d740f3727Mike Hommey#else 324a14bce85e885f83c96116cc5438ae52d740f3727Mike Hommey errno = errnum; 325a14bce85e885f83c96116cc5438ae52d740f3727Mike Hommey#endif 326a14bce85e885f83c96116cc5438ae52d740f3727Mike Hommey} 327a14bce85e885f83c96116cc5438ae52d740f3727Mike Hommey 328e12eaf93dca308a426c182956197b0eeb5f2cff3Jason Evans/* Get last error code. */ 329a14bce85e885f83c96116cc5438ae52d740f3727Mike HommeyJEMALLOC_INLINE int 330a14bce85e885f83c96116cc5438ae52d740f3727Mike Hommeyget_errno(void) 331a14bce85e885f83c96116cc5438ae52d740f3727Mike Hommey{ 332a14bce85e885f83c96116cc5438ae52d740f3727Mike Hommey 333a14bce85e885f83c96116cc5438ae52d740f3727Mike Hommey#ifdef _WIN32 3347cdea3973cab8640d1f44c7638ed5e30ed18be24Mike Hommey return (GetLastError()); 335a14bce85e885f83c96116cc5438ae52d740f3727Mike Hommey#else 3367cdea3973cab8640d1f44c7638ed5e30ed18be24Mike Hommey return (errno); 337a14bce85e885f83c96116cc5438ae52d740f3727Mike Hommey#endif 338a14bce85e885f83c96116cc5438ae52d740f3727Mike Hommey} 339d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans#endif 340d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans 341d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans#endif /* JEMALLOC_H_INLINES */ 342d81e4bdd5c991bd5642c8b859ef1f752b51cd9beJason Evans/******************************************************************************/ 343