xref: /external/jemalloc/include/jemalloc/internal/jemalloc_internal.h.in

#include <sys/mman.h>
#include <sys/param.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/sysctl.h>
#include <sys/uio.h>

#include <errno.h>
#include <limits.h>
#ifndef SIZE_T_MAX
#  define SIZE_T_MAX	SIZE_MAX
#endif
#include <pthread.h>
#include <sched.h>
#include <stdarg.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <stddef.h>
#ifndef offsetof
#  define offsetof(type, member)	((size_t)&(((type *)NULL)->member))
#endif
#include <inttypes.h>
#include <string.h>
#include <strings.h>
#include <ctype.h>
#include <unistd.h>
#include <fcntl.h>
#include <pthread.h>
#include <math.h>

#define	JEMALLOC_NO_DEMANGLE
#include "../jemalloc@install_suffix@.h"

#include "jemalloc/internal/private_namespace.h"

#ifdef JEMALLOC_CC_SILENCE
#define	UNUSED JEMALLOC_ATTR(unused)
#else
#define	UNUSED
#endif

static const bool config_debug =
#ifdef JEMALLOC_DEBUG
    true
#else
    false
#endif
    ;
static const bool config_dss =
#ifdef JEMALLOC_DSS
    true
#else
    false
#endif
    ;
static const bool config_dynamic_page_shift =
#ifdef JEMALLOC_DYNAMIC_PAGE_SHIFT
    true
#else
    false
#endif
    ;
static const bool config_fill =
#ifdef JEMALLOC_FILL
    true
#else
    false
#endif
    ;
static const bool config_lazy_lock =
#ifdef JEMALLOC_LAZY_LOCK
    true
#else
    false
#endif
    ;
static const bool config_prof =
#ifdef JEMALLOC_PROF
    true
#else
    false
#endif
    ;
static const bool config_prof_libgcc =
#ifdef JEMALLOC_PROF_LIBGCC
    true
#else
    false
#endif
    ;
static const bool config_prof_libunwind =
#ifdef JEMALLOC_PROF_LIBUNWIND
    true
#else
    false
#endif
    ;
static const bool config_stats =
#ifdef JEMALLOC_STATS
    true
#else
    false
#endif
    ;
static const bool config_tcache =
#ifdef JEMALLOC_TCACHE
    true
#else
    false
#endif
    ;
static const bool config_tls =
#ifdef JEMALLOC_TLS
    true
#else
    false
#endif
    ;
static const bool config_xmalloc =
#ifdef JEMALLOC_XMALLOC
    true
#else
    false
#endif
    ;
static const bool config_ivsalloc =
#ifdef JEMALLOC_IVSALLOC
    true
#else
    false
#endif
    ;

#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN))
#include <libkern/OSAtomic.h>
#endif

#ifdef JEMALLOC_ZONE
#include <mach/mach_error.h>
#include <mach/mach_init.h>
#include <mach/vm_map.h>
#include <malloc/malloc.h>
#endif

#define	RB_COMPACT
#include "jemalloc/internal/rb.h"
#include "jemalloc/internal/qr.h"
#include "jemalloc/internal/ql.h"

extern void	(*je_malloc_message)(void *wcbopaque, const char *s);

/*
 * Define a custom assert() in order to reduce the chances of deadlock during
 * assertion failure.
 */
#ifndef assert
#  ifdef JEMALLOC_DEBUG
#    define assert(e) do {						\
	if (!(e)) {							\
		char line_buf[UMAX2S_BUFSIZE];				\
		malloc_write("<jemalloc>: ");				\
		malloc_write(__FILE__);					\
		malloc_write(":");					\
		malloc_write(u2s(__LINE__, 10, line_buf));		\
		malloc_write(": Failed assertion: ");			\
		malloc_write("\"");					\
		malloc_write(#e);					\
		malloc_write("\"\n");					\
		abort();						\
	}								\
} while (0)
#  else
#    define assert(e)
#  endif
#endif

/* Use to assert a particular configuration, e.g., cassert(config_debug). */
#define	cassert(c) do {							\
	if ((c) == false)						\
		assert(false);						\
} while (0)

/*
 * jemalloc can conceptually be broken into components (arena, tcache, etc.),
 * but there are circular dependencies that cannot be broken without
 * substantial performance degradation.  In order to reduce the effect on
 * visual code flow, read the header files in multiple passes, with one of the
 * following cpp variables defined during each pass:
 *
 *   JEMALLOC_H_TYPES   : Preprocessor-defined constants and psuedo-opaque data
 *                        types.
 *   JEMALLOC_H_STRUCTS : Data structures.
 *   JEMALLOC_H_EXTERNS : Extern data declarations and function prototypes.
 *   JEMALLOC_H_INLINES : Inline functions.
 */
/******************************************************************************/
#define JEMALLOC_H_TYPES

#define	ALLOCM_LG_ALIGN_MASK	((int)0x3f)

#define	ZU(z)	((size_t)z)

#ifndef __DECONST
#  define	__DECONST(type, var)	((type)(uintptr_t)(const void *)(var))
#endif

#ifdef JEMALLOC_DEBUG
   /* Disable inlining to make debugging easier. */
#  define JEMALLOC_INLINE
#  define inline
#else
#  define JEMALLOC_ENABLE_INLINE
#  define JEMALLOC_INLINE static inline
#endif

/* Size of stack-allocated buffer passed to buferror(). */
#define	BUFERROR_BUF		64

/* Smallest size class to support. */
#define	LG_TINY_MIN		3
#define	TINY_MIN		(1U << LG_TINY_MIN)

/*
 * Minimum alignment of allocations is 2^LG_QUANTUM bytes (ignoring tiny size
 * classes).
 */
#ifndef LG_QUANTUM
#  ifdef __i386__
#    define LG_QUANTUM		4
#  endif
#  ifdef __ia64__
#    define LG_QUANTUM		4
#  endif
#  ifdef __alpha__
#    define LG_QUANTUM		4
#  endif
#  ifdef __sparc64__
#    define LG_QUANTUM		4
#  endif
#  if (defined(__amd64__) || defined(__x86_64__))
#    define LG_QUANTUM		4
#  endif
#  ifdef __arm__
#    define LG_QUANTUM		3
#  endif
#  ifdef __mips__
#    define LG_QUANTUM		3
#  endif
#  ifdef __powerpc__
#    define LG_QUANTUM		4
#  endif
#  ifdef __s390x__
#    define LG_QUANTUM		4
#  endif
#  ifdef __tile__
#    define LG_QUANTUM		4
#  endif
#  ifndef LG_QUANTUM
#    error "No LG_QUANTUM definition for architecture; specify via CPPFLAGS"
#  endif
#endif

#define	QUANTUM			((size_t)(1U << LG_QUANTUM))
#define	QUANTUM_MASK		(QUANTUM - 1)

/* Return the smallest quantum multiple that is >= a. */
#define	QUANTUM_CEILING(a)						\
	(((a) + QUANTUM_MASK) & ~QUANTUM_MASK)

#define	LONG			((size_t)(1U << LG_SIZEOF_LONG))
#define	LONG_MASK		(LONG - 1)

/* Return the smallest long multiple that is >= a. */
#define	LONG_CEILING(a)						\
	(((a) + LONG_MASK) & ~LONG_MASK)

#define	SIZEOF_PTR		(1U << LG_SIZEOF_PTR)
#define	PTR_MASK		(SIZEOF_PTR - 1)

/* Return the smallest (void *) multiple that is >= a. */
#define	PTR_CEILING(a)						\
	(((a) + PTR_MASK) & ~PTR_MASK)

/*
 * Maximum size of L1 cache line.  This is used to avoid cache line aliasing.
 * In addition, this controls the spacing of cacheline-spaced size classes.
 */
#define	LG_CACHELINE		6
#define	CACHELINE		((size_t)(1U << LG_CACHELINE))
#define	CACHELINE_MASK		(CACHELINE - 1)

/* Return the smallest cacheline multiple that is >= s. */
#define	CACHELINE_CEILING(s)						\
	(((s) + CACHELINE_MASK) & ~CACHELINE_MASK)

/* Page size.  STATIC_PAGE_SHIFT is determined by the configure script. */
#define	STATIC_PAGE_SIZE ((size_t)(1U << STATIC_PAGE_SHIFT))
#define	STATIC_PAGE_MASK ((size_t)(STATIC_PAGE_SIZE - 1))
#ifdef PAGE_SHIFT
#  undef PAGE_SHIFT
#endif
#ifdef PAGE_SIZE
#  undef PAGE_SIZE
#endif
#ifdef PAGE_MASK
#  undef PAGE_MASK
#endif
#define	PAGE_SHIFT	STATIC_PAGE_SHIFT
#define	PAGE_SIZE	STATIC_PAGE_SIZE
#define	PAGE_MASK	STATIC_PAGE_MASK

/* Return the smallest pagesize multiple that is >= s. */
#define	PAGE_CEILING(s)							\
	(((s) + PAGE_MASK) & ~PAGE_MASK)

#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/prng.h"
#include "jemalloc/internal/ckh.h"
#include "jemalloc/internal/size_classes.h"
#include "jemalloc/internal/stats.h"
#include "jemalloc/internal/ctl.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/mb.h"
#include "jemalloc/internal/extent.h"
#include "jemalloc/internal/arena.h"
#include "jemalloc/internal/bitmap.h"
#include "jemalloc/internal/base.h"
#include "jemalloc/internal/chunk.h"
#include "jemalloc/internal/huge.h"
#include "jemalloc/internal/rtree.h"
#include "jemalloc/internal/tcache.h"
#include "jemalloc/internal/hash.h"
#ifdef JEMALLOC_ZONE
#include "jemalloc/internal/zone.h"
#endif
#include "jemalloc/internal/prof.h"

#undef JEMALLOC_H_TYPES
/******************************************************************************/
#define JEMALLOC_H_STRUCTS

#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/prng.h"
#include "jemalloc/internal/ckh.h"
#include "jemalloc/internal/size_classes.h"
#include "jemalloc/internal/stats.h"
#include "jemalloc/internal/ctl.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/mb.h"
#include "jemalloc/internal/bitmap.h"
#include "jemalloc/internal/extent.h"
#include "jemalloc/internal/arena.h"
#include "jemalloc/internal/base.h"
#include "jemalloc/internal/chunk.h"
#include "jemalloc/internal/huge.h"
#include "jemalloc/internal/rtree.h"
#include "jemalloc/internal/tcache.h"
#include "jemalloc/internal/hash.h"
#ifdef JEMALLOC_ZONE
#include "jemalloc/internal/zone.h"
#endif
#include "jemalloc/internal/prof.h"

typedef struct {
	uint64_t	allocated;
	uint64_t	deallocated;
} thread_allocated_t;

#undef JEMALLOC_H_STRUCTS
/******************************************************************************/
#define JEMALLOC_H_EXTERNS

extern bool	opt_abort;
extern bool	opt_junk;
extern bool	opt_xmalloc;
extern bool	opt_zero;
extern size_t	opt_narenas;

#ifdef DYNAMIC_PAGE_SHIFT
extern size_t		pagesize;
extern size_t		pagesize_mask;
extern size_t		lg_pagesize;
#endif

/* Number of CPUs. */
extern unsigned		ncpus;

extern malloc_mutex_t	arenas_lock; /* Protects arenas initialization. */
extern pthread_key_t	arenas_tsd;
#ifndef NO_TLS
/*
 * Map of pthread_self() --> arenas[???], used for selecting an arena to use
 * for allocations.
 */
extern __thread arena_t	*arenas_tls JEMALLOC_ATTR(tls_model("initial-exec"));
#  define ARENA_GET()	arenas_tls
#  define ARENA_SET(v)	do {						\
	arenas_tls = (v);						\
	pthread_setspecific(arenas_tsd, (void *)(v));			\
} while (0)
#else
#  define ARENA_GET()	((arena_t *)pthread_getspecific(arenas_tsd))
#  define ARENA_SET(v)	do {						\
	pthread_setspecific(arenas_tsd, (void *)(v));			\
} while (0)
#endif

/*
 * Arenas that are used to service external requests.  Not all elements of the
 * arenas array are necessarily used; arenas are created lazily as needed.
 */
extern arena_t		**arenas;
extern unsigned		narenas;

#ifndef NO_TLS
extern __thread thread_allocated_t	thread_allocated_tls;
#    define ALLOCATED_GET() (thread_allocated_tls.allocated)
#    define ALLOCATEDP_GET() (&thread_allocated_tls.allocated)
#    define DEALLOCATED_GET() (thread_allocated_tls.deallocated)
#    define DEALLOCATEDP_GET() (&thread_allocated_tls.deallocated)
#    define ALLOCATED_ADD(a, d) do {					\
	thread_allocated_tls.allocated += a;				\
	thread_allocated_tls.deallocated += d;				\
} while (0)
#else
#    define ALLOCATED_GET() (thread_allocated_get()->allocated)
#    define ALLOCATEDP_GET() (&thread_allocated_get()->allocated)
#    define DEALLOCATED_GET() (thread_allocated_get()->deallocated)
#    define DEALLOCATEDP_GET() (&thread_allocated_get()->deallocated)
#    define ALLOCATED_ADD(a, d) do {					\
	thread_allocated_t *thread_allocated = thread_allocated_get();	\
	thread_allocated->allocated += (a);				\
	thread_allocated->deallocated += (d);				\
} while (0)
#endif
extern pthread_key_t	thread_allocated_tsd;
thread_allocated_t	*thread_allocated_get_hard(void);

arena_t	*arenas_extend(unsigned ind);
arena_t	*choose_arena_hard(void);
int	buferror(int errnum, char *buf, size_t buflen);
void	jemalloc_prefork(void);
void	jemalloc_postfork(void);

#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/prng.h"
#include "jemalloc/internal/ckh.h"
#include "jemalloc/internal/size_classes.h"
#include "jemalloc/internal/stats.h"
#include "jemalloc/internal/ctl.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/mb.h"
#include "jemalloc/internal/bitmap.h"
#include "jemalloc/internal/extent.h"
#include "jemalloc/internal/arena.h"
#include "jemalloc/internal/base.h"
#include "jemalloc/internal/chunk.h"
#include "jemalloc/internal/huge.h"
#include "jemalloc/internal/rtree.h"
#include "jemalloc/internal/tcache.h"
#include "jemalloc/internal/hash.h"
#ifdef JEMALLOC_ZONE
#include "jemalloc/internal/zone.h"
#endif
#include "jemalloc/internal/prof.h"

#undef JEMALLOC_H_EXTERNS
/******************************************************************************/
#define JEMALLOC_H_INLINES

#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/prng.h"
#include "jemalloc/internal/ckh.h"
#include "jemalloc/internal/size_classes.h"
#include "jemalloc/internal/stats.h"
#include "jemalloc/internal/ctl.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/mb.h"
#include "jemalloc/internal/extent.h"
#include "jemalloc/internal/base.h"
#include "jemalloc/internal/chunk.h"
#include "jemalloc/internal/huge.h"

#ifndef JEMALLOC_ENABLE_INLINE
size_t	pow2_ceil(size_t x);
size_t	s2u(size_t size);
size_t	sa2u(size_t size, size_t alignment, size_t *run_size_p);
void	malloc_write(const char *s);
arena_t	*choose_arena(void);
thread_allocated_t	*thread_allocated_get(void);
#endif

#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
/* Compute the smallest power of 2 that is >= x. */
JEMALLOC_INLINE size_t
pow2_ceil(size_t x)
{

	x--;
	x |= x >> 1;
	x |= x >> 2;
	x |= x >> 4;
	x |= x >> 8;
	x |= x >> 16;
#if (LG_SIZEOF_PTR == 3)
	x |= x >> 32;
#endif
	x++;
	return (x);
}

/*
 * Compute usable size that would result from allocating an object with the
 * specified size.
 */
JEMALLOC_INLINE size_t
s2u(size_t size)
{

	if (size <= SMALL_MAXCLASS)
		return (arena_bin_info[SMALL_SIZE2BIN(size)].reg_size);
	if (size <= arena_maxclass)
		return (PAGE_CEILING(size));
	return (CHUNK_CEILING(size));
}

/*
 * Compute usable size that would result from allocating an object with the
 * specified size and alignment.
 */
JEMALLOC_INLINE size_t
sa2u(size_t size, size_t alignment, size_t *run_size_p)
{
	size_t usize;

	/*
	 * Round size up to the nearest multiple of alignment.
	 *
	 * This done, we can take advantage of the fact that for each small
	 * size class, every object is aligned at the smallest power of two
	 * that is non-zero in the base two representation of the size.  For
	 * example:
	 *
	 *   Size |   Base 2 | Minimum alignment
	 *   -----+----------+------------------
	 *     96 |  1100000 |  32
	 *    144 | 10100000 |  32
	 *    192 | 11000000 |  64
	 */
	usize = (size + (alignment - 1)) & (-alignment);
	/*
	 * (usize < size) protects against the combination of maximal
	 * alignment and size greater than maximal alignment.
	 */
	if (usize < size) {
		/* size_t overflow. */
		return (0);
	}

	if (usize <= arena_maxclass && alignment <= PAGE_SIZE) {
		if (usize <= SMALL_MAXCLASS)
			return (arena_bin_info[SMALL_SIZE2BIN(usize)].reg_size);
		return (PAGE_CEILING(usize));
	} else {
		size_t run_size;

		/*
		 * We can't achieve subpage alignment, so round up alignment
		 * permanently; it makes later calculations simpler.
		 */
		alignment = PAGE_CEILING(alignment);
		usize = PAGE_CEILING(size);
		/*
		 * (usize < size) protects against very large sizes within
		 * PAGE_SIZE of SIZE_T_MAX.
		 *
		 * (usize + alignment < usize) protects against the
		 * combination of maximal alignment and usize large enough
		 * to cause overflow.  This is similar to the first overflow
		 * check above, but it needs to be repeated due to the new
		 * usize value, which may now be *equal* to maximal
		 * alignment, whereas before we only detected overflow if the
		 * original size was *greater* than maximal alignment.
		 */
		if (usize < size || usize + alignment < usize) {
			/* size_t overflow. */
			return (0);
		}

		/*
		 * Calculate the size of the over-size run that arena_palloc()
		 * would need to allocate in order to guarantee the alignment.
		 */
		if (usize >= alignment)
			run_size = usize + alignment - PAGE_SIZE;
		else {
			/*
			 * It is possible that (alignment << 1) will cause
			 * overflow, but it doesn't matter because we also
			 * subtract PAGE_SIZE, which in the case of overflow
			 * leaves us with a very large run_size.  That causes
			 * the first conditional below to fail, which means
			 * that the bogus run_size value never gets used for
			 * anything important.
			 */
			run_size = (alignment << 1) - PAGE_SIZE;
		}
		if (run_size_p != NULL)
			*run_size_p = run_size;

		if (run_size <= arena_maxclass)
			return (PAGE_CEILING(usize));
		return (CHUNK_CEILING(usize));
	}
}

/*
 * Wrapper around malloc_message() that avoids the need for
 * je_malloc_message(...) throughout the code.
 */
JEMALLOC_INLINE void
malloc_write(const char *s)
{

	je_malloc_message(NULL, s);
}

/*
 * Choose an arena based on a per-thread value (fast-path code, calls slow-path
 * code if necessary).
 */
JEMALLOC_INLINE arena_t *
choose_arena(void)
{
	arena_t *ret;

	ret = ARENA_GET();
	if (ret == NULL) {
		ret = choose_arena_hard();
		assert(ret != NULL);
	}

	return (ret);
}

JEMALLOC_INLINE thread_allocated_t *
thread_allocated_get(void)
{
	thread_allocated_t *thread_allocated = (thread_allocated_t *)
	    pthread_getspecific(thread_allocated_tsd);

	if (thread_allocated == NULL)
		return (thread_allocated_get_hard());
	return (thread_allocated);
}
#endif

#include "jemalloc/internal/bitmap.h"
#include "jemalloc/internal/rtree.h"
#include "jemalloc/internal/tcache.h"
#include "jemalloc/internal/arena.h"
#include "jemalloc/internal/hash.h"
#ifdef JEMALLOC_ZONE
#include "jemalloc/internal/zone.h"
#endif

#ifndef JEMALLOC_ENABLE_INLINE
void	*imalloc(size_t size);
void	*icalloc(size_t size);
void	*ipalloc(size_t usize, size_t alignment, bool zero);
size_t	isalloc(const void *ptr);
size_t	ivsalloc(const void *ptr);
void	idalloc(void *ptr);
void	*iralloc(void *ptr, size_t size, size_t extra, size_t alignment,
    bool zero, bool no_move);
#endif

#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
JEMALLOC_INLINE void *
imalloc(size_t size)
{

	assert(size != 0);

	if (size <= arena_maxclass)
		return (arena_malloc(size, false));
	else
		return (huge_malloc(size, false));
}

JEMALLOC_INLINE void *
icalloc(size_t size)
{

	if (size <= arena_maxclass)
		return (arena_malloc(size, true));
	else
		return (huge_malloc(size, true));
}

JEMALLOC_INLINE void *
ipalloc(size_t usize, size_t alignment, bool zero)
{
	void *ret;

	assert(usize != 0);
	assert(usize == sa2u(usize, alignment, NULL));

	if (usize <= arena_maxclass && alignment <= PAGE_SIZE)
		ret = arena_malloc(usize, zero);
	else {
		size_t run_size
#ifdef JEMALLOC_CC_SILENCE
		    = 0
#endif
		    ;

		/*
		 * Ideally we would only ever call sa2u() once per aligned
		 * allocation request, and the caller of this function has
		 * already done so once.  However, it's rather burdensome to
		 * require every caller to pass in run_size, especially given
		 * that it's only relevant to large allocations.  Therefore,
		 * just call it again here in order to get run_size.
		 */
		sa2u(usize, alignment, &run_size);
		if (run_size <= arena_maxclass) {
			ret = arena_palloc(choose_arena(), usize, run_size,
			    alignment, zero);
		} else if (alignment <= chunksize)
			ret = huge_malloc(usize, zero);
		else
			ret = huge_palloc(usize, alignment, zero);
	}

	assert(((uintptr_t)ret & (alignment - 1)) == 0);
	return (ret);
}

JEMALLOC_INLINE size_t
isalloc(const void *ptr)
{
	size_t ret;
	arena_chunk_t *chunk;

	assert(ptr != NULL);

	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
	if (chunk != ptr) {
		/* Region. */
		if (config_prof)
			ret = arena_salloc_demote(ptr);
		else
			ret = arena_salloc(ptr);
	} else
		ret = huge_salloc(ptr);

	return (ret);
}

JEMALLOC_INLINE size_t
ivsalloc(const void *ptr)
{

	/* Return 0 if ptr is not within a chunk managed by jemalloc. */
	if (rtree_get(chunks_rtree, (uintptr_t)CHUNK_ADDR2BASE(ptr)) == NULL)
		return (0);

	return (isalloc(ptr));
}

JEMALLOC_INLINE void
idalloc(void *ptr)
{
	arena_chunk_t *chunk;

	assert(ptr != NULL);

	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
	if (chunk != ptr)
		arena_dalloc(chunk->arena, chunk, ptr);
	else
		huge_dalloc(ptr, true);
}

JEMALLOC_INLINE void *
iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
    bool no_move)
{
	void *ret;
	size_t oldsize;

	assert(ptr != NULL);
	assert(size != 0);

	oldsize = isalloc(ptr);

	if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1))
	    != 0) {
		size_t usize, copysize;

		/*
		 * Existing object alignment is inadquate; allocate new space
		 * and copy.
		 */
		if (no_move)
			return (NULL);
		usize = sa2u(size + extra, alignment, NULL);
		if (usize == 0)
			return (NULL);
		ret = ipalloc(usize, alignment, zero);
		if (ret == NULL) {
			if (extra == 0)
				return (NULL);
			/* Try again, without extra this time. */
			usize = sa2u(size, alignment, NULL);
			if (usize == 0)
				return (NULL);
			ret = ipalloc(usize, alignment, zero);
			if (ret == NULL)
				return (NULL);
		}
		/*
		 * Copy at most size bytes (not size+extra), since the caller
		 * has no expectation that the extra bytes will be reliably
		 * preserved.
		 */
		copysize = (size < oldsize) ? size : oldsize;
		memcpy(ret, ptr, copysize);
		idalloc(ptr);
		return (ret);
	}

	if (no_move) {
		if (size <= arena_maxclass) {
			return (arena_ralloc_no_move(ptr, oldsize, size,
			    extra, zero));
		} else {
			return (huge_ralloc_no_move(ptr, oldsize, size,
			    extra));
		}
	} else {
		if (size + extra <= arena_maxclass) {
			return (arena_ralloc(ptr, oldsize, size, extra,
			    alignment, zero));
		} else {
			return (huge_ralloc(ptr, oldsize, size, extra,
			    alignment, zero));
		}
	}
}
#endif

#include "jemalloc/internal/prof.h"

#undef JEMALLOC_H_INLINES
/******************************************************************************/