1/******************************************************************************/
2#ifdef JEMALLOC_H_TYPES
3
4#ifdef _WIN32
5#  ifdef _WIN64
6#    define FMT64_PREFIX "ll"
7#    define FMTPTR_PREFIX "ll"
8#  else
9#    define FMT64_PREFIX "ll"
10#    define FMTPTR_PREFIX ""
11#  endif
12#  define FMTd32 "d"
13#  define FMTu32 "u"
14#  define FMTx32 "x"
15#  define FMTd64 FMT64_PREFIX "d"
16#  define FMTu64 FMT64_PREFIX "u"
17#  define FMTx64 FMT64_PREFIX "x"
18#  define FMTdPTR FMTPTR_PREFIX "d"
19#  define FMTuPTR FMTPTR_PREFIX "u"
20#  define FMTxPTR FMTPTR_PREFIX "x"
21#else
22#  include <inttypes.h>
23#  define FMTd32 PRId32
24#  define FMTu32 PRIu32
25#  define FMTx32 PRIx32
26#  define FMTd64 PRId64
27#  define FMTu64 PRIu64
28#  define FMTx64 PRIx64
29#  define FMTdPTR PRIdPTR
30#  define FMTuPTR PRIuPTR
31#  define FMTxPTR PRIxPTR
32#endif
33
34/* Size of stack-allocated buffer passed to buferror(). */
35#define	BUFERROR_BUF		64
36
37/*
38 * Size of stack-allocated buffer used by malloc_{,v,vc}printf().  This must be
39 * large enough for all possible uses within jemalloc.
40 */
41#define	MALLOC_PRINTF_BUFSIZE	4096
42
43/*
44 * Wrap a cpp argument that contains commas such that it isn't broken up into
45 * multiple arguments.
46 */
47#define	JEMALLOC_ARG_CONCAT(...) __VA_ARGS__
48
49/*
50 * Silence compiler warnings due to uninitialized values.  This is used
51 * wherever the compiler fails to recognize that the variable is never used
52 * uninitialized.
53 */
54#ifdef JEMALLOC_CC_SILENCE
55#	define JEMALLOC_CC_SILENCE_INIT(v) = v
56#else
57#	define JEMALLOC_CC_SILENCE_INIT(v)
58#endif
59
60#define	JEMALLOC_GNUC_PREREQ(major, minor)				\
61    (!defined(__clang__) &&						\
62    (__GNUC__ > (major) || (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor))))
63#ifndef __has_builtin
64#  define __has_builtin(builtin) (0)
65#endif
66#define	JEMALLOC_CLANG_HAS_BUILTIN(builtin)				\
67    (defined(__clang__) && __has_builtin(builtin))
68
69#ifdef __GNUC__
70#	define likely(x)   __builtin_expect(!!(x), 1)
71#	define unlikely(x) __builtin_expect(!!(x), 0)
72#  if JEMALLOC_GNUC_PREREQ(4, 6) ||					\
73      JEMALLOC_CLANG_HAS_BUILTIN(__builtin_unreachable)
74#	define unreachable() __builtin_unreachable()
75#  else
76#	define unreachable()
77#  endif
78#else
79#	define likely(x)   !!(x)
80#	define unlikely(x) !!(x)
81#	define unreachable()
82#endif
83
84#include "jemalloc/internal/assert.h"
85
86/* Use to assert a particular configuration, e.g., cassert(config_debug). */
87#define	cassert(c) do {							\
88	if (unlikely(!(c)))						\
89		not_reached();						\
90} while (0)
91
92#endif /* JEMALLOC_H_TYPES */
93/******************************************************************************/
94#ifdef JEMALLOC_H_STRUCTS
95
96#endif /* JEMALLOC_H_STRUCTS */
97/******************************************************************************/
98#ifdef JEMALLOC_H_EXTERNS
99
100int	buferror(int err, char *buf, size_t buflen);
101uintmax_t	malloc_strtoumax(const char *restrict nptr,
102    char **restrict endptr, int base);
103void	malloc_write(const char *s);
104
105/*
106 * malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating
107 * point math.
108 */
109int	malloc_vsnprintf(char *str, size_t size, const char *format,
110    va_list ap);
111int	malloc_snprintf(char *str, size_t size, const char *format, ...)
112    JEMALLOC_FORMAT_PRINTF(3, 4);
113void	malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
114    const char *format, va_list ap);
115void malloc_cprintf(void (*write)(void *, const char *), void *cbopaque,
116    const char *format, ...) JEMALLOC_FORMAT_PRINTF(3, 4);
117void	malloc_printf(const char *format, ...) JEMALLOC_FORMAT_PRINTF(1, 2);
118
119#endif /* JEMALLOC_H_EXTERNS */
120/******************************************************************************/
121#ifdef JEMALLOC_H_INLINES
122
123#ifndef JEMALLOC_ENABLE_INLINE
124unsigned	ffs_llu(unsigned long long bitmap);
125unsigned	ffs_lu(unsigned long bitmap);
126unsigned	ffs_u(unsigned bitmap);
127unsigned	ffs_zu(size_t bitmap);
128unsigned	ffs_u64(uint64_t bitmap);
129unsigned	ffs_u32(uint32_t bitmap);
130uint64_t	pow2_ceil_u64(uint64_t x);
131uint32_t	pow2_ceil_u32(uint32_t x);
132size_t	pow2_ceil_zu(size_t x);
133unsigned	lg_floor(size_t x);
134void	set_errno(int errnum);
135int	get_errno(void);
136#endif
137
138#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_UTIL_C_))
139
140/* Sanity check. */
141#if !defined(JEMALLOC_INTERNAL_FFSLL) || !defined(JEMALLOC_INTERNAL_FFSL) \
142    || !defined(JEMALLOC_INTERNAL_FFS)
143#  error JEMALLOC_INTERNAL_FFS{,L,LL} should have been defined by configure
144#endif
145
146JEMALLOC_ALWAYS_INLINE unsigned
147ffs_llu(unsigned long long bitmap)
148{
149
150	return (JEMALLOC_INTERNAL_FFSLL(bitmap));
151}
152
153JEMALLOC_ALWAYS_INLINE unsigned
154ffs_lu(unsigned long bitmap)
155{
156
157	return (JEMALLOC_INTERNAL_FFSL(bitmap));
158}
159
160JEMALLOC_ALWAYS_INLINE unsigned
161ffs_u(unsigned bitmap)
162{
163
164	return (JEMALLOC_INTERNAL_FFS(bitmap));
165}
166
167JEMALLOC_ALWAYS_INLINE unsigned
168ffs_zu(size_t bitmap)
169{
170
171#if LG_SIZEOF_PTR == LG_SIZEOF_INT
172	return (ffs_u(bitmap));
173#elif LG_SIZEOF_PTR == LG_SIZEOF_LONG
174	return (ffs_lu(bitmap));
175#elif LG_SIZEOF_PTR == LG_SIZEOF_LONG_LONG
176	return (ffs_llu(bitmap));
177#else
178#error No implementation for size_t ffs()
179#endif
180}
181
182JEMALLOC_ALWAYS_INLINE unsigned
183ffs_u64(uint64_t bitmap)
184{
185
186#if LG_SIZEOF_LONG == 3
187	return (ffs_lu(bitmap));
188#elif LG_SIZEOF_LONG_LONG == 3
189	return (ffs_llu(bitmap));
190#else
191#error No implementation for 64-bit ffs()
192#endif
193}
194
195JEMALLOC_ALWAYS_INLINE unsigned
196ffs_u32(uint32_t bitmap)
197{
198
199#if LG_SIZEOF_INT == 2
200	return (ffs_u(bitmap));
201#else
202#error No implementation for 32-bit ffs()
203#endif
204	return (ffs_u(bitmap));
205}
206
207JEMALLOC_INLINE uint64_t
208pow2_ceil_u64(uint64_t x)
209{
210
211	x--;
212	x |= x >> 1;
213	x |= x >> 2;
214	x |= x >> 4;
215	x |= x >> 8;
216	x |= x >> 16;
217	x |= x >> 32;
218	x++;
219	return (x);
220}
221
222JEMALLOC_INLINE uint32_t
223pow2_ceil_u32(uint32_t x)
224{
225
226	x--;
227	x |= x >> 1;
228	x |= x >> 2;
229	x |= x >> 4;
230	x |= x >> 8;
231	x |= x >> 16;
232	x++;
233	return (x);
234}
235
236/* Compute the smallest power of 2 that is >= x. */
237JEMALLOC_INLINE size_t
238pow2_ceil_zu(size_t x)
239{
240
241#if (LG_SIZEOF_PTR == 3)
242	return (pow2_ceil_u64(x));
243#else
244	return (pow2_ceil_u32(x));
245#endif
246}
247
248#if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__))
249JEMALLOC_INLINE unsigned
250lg_floor(size_t x)
251{
252	size_t ret;
253
254	assert(x != 0);
255
256	asm ("bsr %1, %0"
257	    : "=r"(ret) // Outputs.
258	    : "r"(x)    // Inputs.
259	    );
260	assert(ret < UINT_MAX);
261	return ((unsigned)ret);
262}
263#elif (defined(_MSC_VER))
264JEMALLOC_INLINE unsigned
265lg_floor(size_t x)
266{
267	unsigned long ret;
268
269	assert(x != 0);
270
271#if (LG_SIZEOF_PTR == 3)
272	_BitScanReverse64(&ret, x);
273#elif (LG_SIZEOF_PTR == 2)
274	_BitScanReverse(&ret, x);
275#else
276#  error "Unsupported type size for lg_floor()"
277#endif
278	assert(ret < UINT_MAX);
279	return ((unsigned)ret);
280}
281#elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ))
282JEMALLOC_INLINE unsigned
283lg_floor(size_t x)
284{
285
286	assert(x != 0);
287
288#if (LG_SIZEOF_PTR == LG_SIZEOF_INT)
289	return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clz(x));
290#elif (LG_SIZEOF_PTR == LG_SIZEOF_LONG)
291	return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clzl(x));
292#else
293#  error "Unsupported type size for lg_floor()"
294#endif
295}
296#else
297JEMALLOC_INLINE unsigned
298lg_floor(size_t x)
299{
300
301	assert(x != 0);
302
303	x |= (x >> 1);
304	x |= (x >> 2);
305	x |= (x >> 4);
306	x |= (x >> 8);
307	x |= (x >> 16);
308#if (LG_SIZEOF_PTR == 3)
309	x |= (x >> 32);
310#endif
311	if (x == SIZE_T_MAX)
312		return ((8 << LG_SIZEOF_PTR) - 1);
313	x++;
314	return (ffs_zu(x) - 2);
315}
316#endif
317
318/* Set error code. */
319JEMALLOC_INLINE void
320set_errno(int errnum)
321{
322
323#ifdef _WIN32
324	SetLastError(errnum);
325#else
326	errno = errnum;
327#endif
328}
329
330/* Get last error code. */
331JEMALLOC_INLINE int
332get_errno(void)
333{
334
335#ifdef _WIN32
336	return (GetLastError());
337#else
338	return (errno);
339#endif
340}
341#endif
342
343#endif /* JEMALLOC_H_INLINES */
344/******************************************************************************/
345