jemalloc.c revision ecd3e59ca351d7111ec72a327fe0c009f2aa69a0 
1#define	JEMALLOC_C_
2#include "jemalloc/internal/jemalloc_internal.h"
3
4/******************************************************************************/
5/* Data. */
6
7malloc_tsd_data(, arenas, arena_t *, NULL)
8malloc_tsd_data(, thread_allocated, thread_allocated_t,
9    THREAD_ALLOCATED_INITIALIZER)
10
11/* Runtime configuration options. */
12const char	*je_malloc_conf;
13bool	opt_abort =
14#ifdef JEMALLOC_DEBUG
15    true
16#else
17    false
18#endif
19    ;
20bool	opt_junk =
21#if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
22    true
23#else
24    false
25#endif
26    ;
27size_t	opt_quarantine = ZU(0);
28bool	opt_redzone = false;
29bool	opt_utrace = false;
30bool	opt_xmalloc = false;
31bool	opt_zero = false;
32size_t	opt_narenas = 0;
33
34/* Initialized to true if the process is running inside Valgrind. */
35bool	in_valgrind;
36
37unsigned	ncpus;
38
39malloc_mutex_t		arenas_lock;
40arena_t			**arenas;
41unsigned		narenas_total;
42unsigned		narenas_auto;
43
44/* Set to true once the allocator has been initialized. */
45static bool		malloc_initialized = false;
46
47#ifdef JEMALLOC_THREADED_INIT
48/* Used to let the initializing thread recursively allocate. */
49#  define NO_INITIALIZER	((unsigned long)0)
50#  define INITIALIZER		pthread_self()
51#  define IS_INITIALIZER	(malloc_initializer == pthread_self())
52static pthread_t		malloc_initializer = NO_INITIALIZER;
53#else
54#  define NO_INITIALIZER	false
55#  define INITIALIZER		true
56#  define IS_INITIALIZER	malloc_initializer
57static bool			malloc_initializer = NO_INITIALIZER;
58#endif
59
60/* Used to avoid initialization races. */
61#ifdef _WIN32
62static malloc_mutex_t	init_lock;
63
64JEMALLOC_ATTR(constructor)
65static void WINAPI
66_init_init_lock(void)
67{
68
69	malloc_mutex_init(&init_lock);
70}
71
72#ifdef _MSC_VER
73#  pragma section(".CRT$XCU", read)
74JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used)
75static const void (WINAPI *init_init_lock)(void) = _init_init_lock;
76#endif
77
78#else
79static malloc_mutex_t	init_lock = MALLOC_MUTEX_INITIALIZER;
80#endif
81
82typedef struct {
83	void	*p;	/* Input pointer (as in realloc(p, s)). */
84	size_t	s;	/* Request size. */
85	void	*r;	/* Result pointer. */
86} malloc_utrace_t;
87
88#ifdef JEMALLOC_UTRACE
89#  define UTRACE(a, b, c) do {						\
90	if (opt_utrace) {						\
91		int utrace_serrno = errno;				\
92		malloc_utrace_t ut;					\
93		ut.p = (a);						\
94		ut.s = (b);						\
95		ut.r = (c);						\
96		utrace(&ut, sizeof(ut));				\
97		errno = utrace_serrno;					\
98	}								\
99} while (0)
100#else
101#  define UTRACE(a, b, c)
102#endif
103
104/******************************************************************************/
105/*
106 * Function prototypes for static functions that are referenced prior to
107 * definition.
108 */
109
110static bool	malloc_init_hard(void);
111
112/******************************************************************************/
113/*
114 * Begin miscellaneous support functions.
115 */
116
117/* Create a new arena and insert it into the arenas array at index ind. */
118arena_t *
119arenas_extend(unsigned ind)
120{
121	arena_t *ret;
122
123	ret = (arena_t *)base_alloc(sizeof(arena_t));
124	if (ret != NULL && arena_new(ret, ind) == false) {
125		arenas[ind] = ret;
126		return (ret);
127	}
128	/* Only reached if there is an OOM error. */
129
130	/*
131	 * OOM here is quite inconvenient to propagate, since dealing with it
132	 * would require a check for failure in the fast path.  Instead, punt
133	 * by using arenas[0].  In practice, this is an extremely unlikely
134	 * failure.
135	 */
136	malloc_write("<jemalloc>: Error initializing arena\n");
137	if (opt_abort)
138		abort();
139
140	return (arenas[0]);
141}
142
143/* Slow path, called only by choose_arena(). */
144arena_t *
145choose_arena_hard(void)
146{
147	arena_t *ret;
148
149	if (narenas_auto > 1) {
150		unsigned i, choose, first_null;
151
152		choose = 0;
153		first_null = narenas_auto;
154		malloc_mutex_lock(&arenas_lock);
155		assert(arenas[0] != NULL);
156		for (i = 1; i < narenas_auto; i++) {
157			if (arenas[i] != NULL) {
158				/*
159				 * Choose the first arena that has the lowest
160				 * number of threads assigned to it.
161				 */
162				if (arenas[i]->nthreads <
163				    arenas[choose]->nthreads)
164					choose = i;
165			} else if (first_null == narenas_auto) {
166				/*
167				 * Record the index of the first uninitialized
168				 * arena, in case all extant arenas are in use.
169				 *
170				 * NB: It is possible for there to be
171				 * discontinuities in terms of initialized
172				 * versus uninitialized arenas, due to the
173				 * "thread.arena" mallctl.
174				 */
175				first_null = i;
176			}
177		}
178
179		if (arenas[choose]->nthreads == 0
180		    || first_null == narenas_auto) {
181			/*
182			 * Use an unloaded arena, or the least loaded arena if
183			 * all arenas are already initialized.
184			 */
185			ret = arenas[choose];
186		} else {
187			/* Initialize a new arena. */
188			ret = arenas_extend(first_null);
189		}
190		ret->nthreads++;
191		malloc_mutex_unlock(&arenas_lock);
192	} else {
193		ret = arenas[0];
194		malloc_mutex_lock(&arenas_lock);
195		ret->nthreads++;
196		malloc_mutex_unlock(&arenas_lock);
197	}
198
199	arenas_tsd_set(&ret);
200
201	return (ret);
202}
203
204static void
205stats_print_atexit(void)
206{
207
208	if (config_tcache && config_stats) {
209		unsigned narenas, i;
210
211		/*
212		 * Merge stats from extant threads.  This is racy, since
213		 * individual threads do not lock when recording tcache stats
214		 * events.  As a consequence, the final stats may be slightly
215		 * out of date by the time they are reported, if other threads
216		 * continue to allocate.
217		 */
218		for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
219			arena_t *arena = arenas[i];
220			if (arena != NULL) {
221				tcache_t *tcache;
222
223				/*
224				 * tcache_stats_merge() locks bins, so if any
225				 * code is introduced that acquires both arena
226				 * and bin locks in the opposite order,
227				 * deadlocks may result.
228				 */
229				malloc_mutex_lock(&arena->lock);
230				ql_foreach(tcache, &arena->tcache_ql, link) {
231					tcache_stats_merge(tcache, arena);
232				}
233				malloc_mutex_unlock(&arena->lock);
234			}
235		}
236	}
237	je_malloc_stats_print(NULL, NULL, NULL);
238}
239
240/*
241 * End miscellaneous support functions.
242 */
243/******************************************************************************/
244/*
245 * Begin initialization functions.
246 */
247
248static unsigned
249malloc_ncpus(void)
250{
251	long result;
252
253#ifdef _WIN32
254	SYSTEM_INFO si;
255	GetSystemInfo(&si);
256	result = si.dwNumberOfProcessors;
257#else
258	result = sysconf(_SC_NPROCESSORS_ONLN);
259#endif
260	return ((result == -1) ? 1 : (unsigned)result);
261}
262
263void
264arenas_cleanup(void *arg)
265{
266	arena_t *arena = *(arena_t **)arg;
267
268	malloc_mutex_lock(&arenas_lock);
269	arena->nthreads--;
270	malloc_mutex_unlock(&arenas_lock);
271}
272
273JEMALLOC_ALWAYS_INLINE_C void
274malloc_thread_init(void)
275{
276
277	/*
278	 * TSD initialization can't be safely done as a side effect of
279	 * deallocation, because it is possible for a thread to do nothing but
280	 * deallocate its TLS data via free(), in which case writing to TLS
281	 * would cause write-after-free memory corruption.  The quarantine
282	 * facility *only* gets used as a side effect of deallocation, so make
283	 * a best effort attempt at initializing its TSD by hooking all
284	 * allocation events.
285	 */
286	if (config_fill && opt_quarantine)
287		quarantine_alloc_hook();
288}
289
290JEMALLOC_ALWAYS_INLINE_C bool
291malloc_init(void)
292{
293
294	if (malloc_initialized == false && malloc_init_hard())
295		return (true);
296	malloc_thread_init();
297
298	return (false);
299}
300
301static bool
302malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p,
303    char const **v_p, size_t *vlen_p)
304{
305	bool accept;
306	const char *opts = *opts_p;
307
308	*k_p = opts;
309
310	for (accept = false; accept == false;) {
311		switch (*opts) {
312		case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
313		case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
314		case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
315		case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
316		case 'Y': case 'Z':
317		case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
318		case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
319		case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
320		case 's': case 't': case 'u': case 'v': case 'w': case 'x':
321		case 'y': case 'z':
322		case '0': case '1': case '2': case '3': case '4': case '5':
323		case '6': case '7': case '8': case '9':
324		case '_':
325			opts++;
326			break;
327		case ':':
328			opts++;
329			*klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p;
330			*v_p = opts;
331			accept = true;
332			break;
333		case '\0':
334			if (opts != *opts_p) {
335				malloc_write("<jemalloc>: Conf string ends "
336				    "with key\n");
337			}
338			return (true);
339		default:
340			malloc_write("<jemalloc>: Malformed conf string\n");
341			return (true);
342		}
343	}
344
345	for (accept = false; accept == false;) {
346		switch (*opts) {
347		case ',':
348			opts++;
349			/*
350			 * Look ahead one character here, because the next time
351			 * this function is called, it will assume that end of
352			 * input has been cleanly reached if no input remains,
353			 * but we have optimistically already consumed the
354			 * comma if one exists.
355			 */
356			if (*opts == '\0') {
357				malloc_write("<jemalloc>: Conf string ends "
358				    "with comma\n");
359			}
360			*vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p;
361			accept = true;
362			break;
363		case '\0':
364			*vlen_p = (uintptr_t)opts - (uintptr_t)*v_p;
365			accept = true;
366			break;
367		default:
368			opts++;
369			break;
370		}
371	}
372
373	*opts_p = opts;
374	return (false);
375}
376
377static void
378malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v,
379    size_t vlen)
380{
381
382	malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k,
383	    (int)vlen, v);
384}
385
386static void
387malloc_conf_init(void)
388{
389	unsigned i;
390	char buf[PATH_MAX + 1];
391	const char *opts, *k, *v;
392	size_t klen, vlen;
393
394	/*
395	 * Automatically configure valgrind before processing options.  The
396	 * valgrind option remains in jemalloc 3.x for compatibility reasons.
397	 */
398	if (config_valgrind) {
399		in_valgrind = (RUNNING_ON_VALGRIND != 0) ? true : false;
400		if (config_fill && in_valgrind) {
401			opt_junk = false;
402			assert(opt_zero == false);
403			opt_quarantine = JEMALLOC_VALGRIND_QUARANTINE_DEFAULT;
404			opt_redzone = true;
405		}
406		if (config_tcache && in_valgrind)
407			opt_tcache = false;
408	}
409
410	for (i = 0; i < 3; i++) {
411		/* Get runtime configuration. */
412		switch (i) {
413		case 0:
414			if (je_malloc_conf != NULL) {
415				/*
416				 * Use options that were compiled into the
417				 * program.
418				 */
419				opts = je_malloc_conf;
420			} else {
421				/* No configuration specified. */
422				buf[0] = '\0';
423				opts = buf;
424			}
425			break;
426		case 1: {
427			int linklen = 0;
428#ifndef _WIN32
429			int saved_errno = errno;
430			const char *linkname =
431#  ifdef JEMALLOC_PREFIX
432			    "/etc/"JEMALLOC_PREFIX"malloc.conf"
433#  else
434			    "/etc/malloc.conf"
435#  endif
436			    ;
437
438			/*
439			 * Try to use the contents of the "/etc/malloc.conf"
440			 * symbolic link's name.
441			 */
442			linklen = readlink(linkname, buf, sizeof(buf) - 1);
443			if (linklen == -1) {
444				/* No configuration specified. */
445				linklen = 0;
446				/* restore errno */
447				set_errno(saved_errno);
448			}
449#endif
450			buf[linklen] = '\0';
451			opts = buf;
452			break;
453		} case 2: {
454			const char *envname =
455#ifdef JEMALLOC_PREFIX
456			    JEMALLOC_CPREFIX"MALLOC_CONF"
457#else
458			    "MALLOC_CONF"
459#endif
460			    ;
461
462			if ((opts = getenv(envname)) != NULL) {
463				/*
464				 * Do nothing; opts is already initialized to
465				 * the value of the MALLOC_CONF environment
466				 * variable.
467				 */
468			} else {
469				/* No configuration specified. */
470				buf[0] = '\0';
471				opts = buf;
472			}
473			break;
474		} default:
475			not_reached();
476			buf[0] = '\0';
477			opts = buf;
478		}
479
480		while (*opts != '\0' && malloc_conf_next(&opts, &k, &klen, &v,
481		    &vlen) == false) {
482#define	CONF_HANDLE_BOOL(o, n)						\
483			if (sizeof(n)-1 == klen && strncmp(n, k,	\
484			    klen) == 0) {				\
485				if (strncmp("true", v, vlen) == 0 &&	\
486				    vlen == sizeof("true")-1)		\
487					o = true;			\
488				else if (strncmp("false", v, vlen) ==	\
489				    0 && vlen == sizeof("false")-1)	\
490					o = false;			\
491				else {					\
492					malloc_conf_error(		\
493					    "Invalid conf value",	\
494					    k, klen, v, vlen);		\
495				}					\
496				continue;				\
497			}
498#define	CONF_HANDLE_SIZE_T(o, n, min, max, clip)			\
499			if (sizeof(n)-1 == klen && strncmp(n, k,	\
500			    klen) == 0) {				\
501				uintmax_t um;				\
502				char *end;				\
503									\
504				set_errno(0);				\
505				um = malloc_strtoumax(v, &end, 0);	\
506				if (get_errno() != 0 || (uintptr_t)end -\
507				    (uintptr_t)v != vlen) {		\
508					malloc_conf_error(		\
509					    "Invalid conf value",	\
510					    k, klen, v, vlen);		\
511				} else if (clip) {			\
512					if (min != 0 && um < min)	\
513						o = min;		\
514					else if (um > max)		\
515						o = max;		\
516					else				\
517						o = um;			\
518				} else {				\
519					if ((min != 0 && um < min) ||	\
520					    um > max) {			\
521						malloc_conf_error(	\
522						    "Out-of-range "	\
523						    "conf value",	\
524						    k, klen, v, vlen);	\
525					} else				\
526						o = um;			\
527				}					\
528				continue;				\
529			}
530#define	CONF_HANDLE_SSIZE_T(o, n, min, max)				\
531			if (sizeof(n)-1 == klen && strncmp(n, k,	\
532			    klen) == 0) {				\
533				long l;					\
534				char *end;				\
535									\
536				set_errno(0);				\
537				l = strtol(v, &end, 0);			\
538				if (get_errno() != 0 || (uintptr_t)end -\
539				    (uintptr_t)v != vlen) {		\
540					malloc_conf_error(		\
541					    "Invalid conf value",	\
542					    k, klen, v, vlen);		\
543				} else if (l < (ssize_t)min || l >	\
544				    (ssize_t)max) {			\
545					malloc_conf_error(		\
546					    "Out-of-range conf value",	\
547					    k, klen, v, vlen);		\
548				} else					\
549					o = l;				\
550				continue;				\
551			}
552#define	CONF_HANDLE_CHAR_P(o, n, d)					\
553			if (sizeof(n)-1 == klen && strncmp(n, k,	\
554			    klen) == 0) {				\
555				size_t cpylen = (vlen <=		\
556				    sizeof(o)-1) ? vlen :		\
557				    sizeof(o)-1;			\
558				strncpy(o, v, cpylen);			\
559				o[cpylen] = '\0';			\
560				continue;				\
561			}
562
563			CONF_HANDLE_BOOL(opt_abort, "abort")
564			/*
565			 * Chunks always require at least one header page, plus
566			 * one data page in the absence of redzones, or three
567			 * pages in the presence of redzones.  In order to
568			 * simplify options processing, fix the limit based on
569			 * config_fill.
570			 */
571			CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE +
572			    (config_fill ? 2 : 1), (sizeof(size_t) << 3) - 1,
573			    true)
574			if (strncmp("dss", k, klen) == 0) {
575				int i;
576				bool match = false;
577				for (i = 0; i < dss_prec_limit; i++) {
578					if (strncmp(dss_prec_names[i], v, vlen)
579					    == 0) {
580						if (chunk_dss_prec_set(i)) {
581							malloc_conf_error(
582							    "Error setting dss",
583							    k, klen, v, vlen);
584						} else {
585							opt_dss =
586							    dss_prec_names[i];
587							match = true;
588							break;
589						}
590					}
591				}
592				if (match == false) {
593					malloc_conf_error("Invalid conf value",
594					    k, klen, v, vlen);
595				}
596				continue;
597			}
598			CONF_HANDLE_SIZE_T(opt_narenas, "narenas", 1,
599			    SIZE_T_MAX, false)
600			CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult",
601			    -1, (sizeof(size_t) << 3) - 1)
602			CONF_HANDLE_BOOL(opt_stats_print, "stats_print")
603			if (config_fill) {
604				CONF_HANDLE_BOOL(opt_junk, "junk")
605				CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine",
606				    0, SIZE_T_MAX, false)
607				CONF_HANDLE_BOOL(opt_redzone, "redzone")
608				CONF_HANDLE_BOOL(opt_zero, "zero")
609			}
610			if (config_utrace) {
611				CONF_HANDLE_BOOL(opt_utrace, "utrace")
612			}
613			if (config_xmalloc) {
614				CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc")
615			}
616			if (config_tcache) {
617				CONF_HANDLE_BOOL(opt_tcache, "tcache")
618				CONF_HANDLE_SSIZE_T(opt_lg_tcache_max,
619				    "lg_tcache_max", -1,
620				    (sizeof(size_t) << 3) - 1)
621			}
622			if (config_prof) {
623				CONF_HANDLE_BOOL(opt_prof, "prof")
624				CONF_HANDLE_CHAR_P(opt_prof_prefix,
625				    "prof_prefix", "jeprof")
626				CONF_HANDLE_BOOL(opt_prof_active, "prof_active")
627				CONF_HANDLE_SSIZE_T(opt_lg_prof_sample,
628				    "lg_prof_sample", 0,
629				    (sizeof(uint64_t) << 3) - 1)
630				CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum")
631				CONF_HANDLE_SSIZE_T(opt_lg_prof_interval,
632				    "lg_prof_interval", -1,
633				    (sizeof(uint64_t) << 3) - 1)
634				CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump")
635				CONF_HANDLE_BOOL(opt_prof_final, "prof_final")
636				CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak")
637			}
638			malloc_conf_error("Invalid conf pair", k, klen, v,
639			    vlen);
640#undef CONF_HANDLE_BOOL
641#undef CONF_HANDLE_SIZE_T
642#undef CONF_HANDLE_SSIZE_T
643#undef CONF_HANDLE_CHAR_P
644		}
645	}
646}
647
648static bool
649malloc_init_hard(void)
650{
651	arena_t *init_arenas[1];
652
653	malloc_mutex_lock(&init_lock);
654	if (malloc_initialized || IS_INITIALIZER) {
655		/*
656		 * Another thread initialized the allocator before this one
657		 * acquired init_lock, or this thread is the initializing
658		 * thread, and it is recursively allocating.
659		 */
660		malloc_mutex_unlock(&init_lock);
661		return (false);
662	}
663#ifdef JEMALLOC_THREADED_INIT
664	if (malloc_initializer != NO_INITIALIZER && IS_INITIALIZER == false) {
665		/* Busy-wait until the initializing thread completes. */
666		do {
667			malloc_mutex_unlock(&init_lock);
668			CPU_SPINWAIT;
669			malloc_mutex_lock(&init_lock);
670		} while (malloc_initialized == false);
671		malloc_mutex_unlock(&init_lock);
672		return (false);
673	}
674#endif
675	malloc_initializer = INITIALIZER;
676
677	malloc_tsd_boot();
678	if (config_prof)
679		prof_boot0();
680
681	malloc_conf_init();
682
683	if (opt_stats_print) {
684		/* Print statistics at exit. */
685		if (atexit(stats_print_atexit) != 0) {
686			malloc_write("<jemalloc>: Error in atexit()\n");
687			if (opt_abort)
688				abort();
689		}
690	}
691
692	if (base_boot()) {
693		malloc_mutex_unlock(&init_lock);
694		return (true);
695	}
696
697	if (chunk_boot()) {
698		malloc_mutex_unlock(&init_lock);
699		return (true);
700	}
701
702	if (ctl_boot()) {
703		malloc_mutex_unlock(&init_lock);
704		return (true);
705	}
706
707	if (config_prof)
708		prof_boot1();
709
710	arena_boot();
711
712	if (config_tcache && tcache_boot0()) {
713		malloc_mutex_unlock(&init_lock);
714		return (true);
715	}
716
717	if (huge_boot()) {
718		malloc_mutex_unlock(&init_lock);
719		return (true);
720	}
721
722	if (malloc_mutex_init(&arenas_lock)) {
723		malloc_mutex_unlock(&init_lock);
724		return (true);
725	}
726
727	/*
728	 * Create enough scaffolding to allow recursive allocation in
729	 * malloc_ncpus().
730	 */
731	narenas_total = narenas_auto = 1;
732	arenas = init_arenas;
733	memset(arenas, 0, sizeof(arena_t *) * narenas_auto);
734
735	/*
736	 * Initialize one arena here.  The rest are lazily created in
737	 * choose_arena_hard().
738	 */
739	arenas_extend(0);
740	if (arenas[0] == NULL) {
741		malloc_mutex_unlock(&init_lock);
742		return (true);
743	}
744
745	/* Initialize allocation counters before any allocations can occur. */
746	if (config_stats && thread_allocated_tsd_boot()) {
747		malloc_mutex_unlock(&init_lock);
748		return (true);
749	}
750
751	if (arenas_tsd_boot()) {
752		malloc_mutex_unlock(&init_lock);
753		return (true);
754	}
755
756	if (config_tcache && tcache_boot1()) {
757		malloc_mutex_unlock(&init_lock);
758		return (true);
759	}
760
761	if (config_fill && quarantine_boot()) {
762		malloc_mutex_unlock(&init_lock);
763		return (true);
764	}
765
766	if (config_prof && prof_boot2()) {
767		malloc_mutex_unlock(&init_lock);
768		return (true);
769	}
770
771	malloc_mutex_unlock(&init_lock);
772	/**********************************************************************/
773	/* Recursive allocation may follow. */
774
775	ncpus = malloc_ncpus();
776
777#if (!defined(JEMALLOC_MUTEX_INIT_CB) && !defined(JEMALLOC_ZONE) \
778    && !defined(_WIN32))
779	/* LinuxThreads's pthread_atfork() allocates. */
780	if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent,
781	    jemalloc_postfork_child) != 0) {
782		malloc_write("<jemalloc>: Error in pthread_atfork()\n");
783		if (opt_abort)
784			abort();
785	}
786#endif
787
788	/* Done recursively allocating. */
789	/**********************************************************************/
790	malloc_mutex_lock(&init_lock);
791
792	if (mutex_boot()) {
793		malloc_mutex_unlock(&init_lock);
794		return (true);
795	}
796
797	if (opt_narenas == 0) {
798		/*
799		 * For SMP systems, create more than one arena per CPU by
800		 * default.
801		 */
802		if (ncpus > 1)
803			opt_narenas = ncpus << 2;
804		else
805			opt_narenas = 1;
806	}
807	narenas_auto = opt_narenas;
808	/*
809	 * Make sure that the arenas array can be allocated.  In practice, this
810	 * limit is enough to allow the allocator to function, but the ctl
811	 * machinery will fail to allocate memory at far lower limits.
812	 */
813	if (narenas_auto > chunksize / sizeof(arena_t *)) {
814		narenas_auto = chunksize / sizeof(arena_t *);
815		malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n",
816		    narenas_auto);
817	}
818	narenas_total = narenas_auto;
819
820	/* Allocate and initialize arenas. */
821	arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas_total);
822	if (arenas == NULL) {
823		malloc_mutex_unlock(&init_lock);
824		return (true);
825	}
826	/*
827	 * Zero the array.  In practice, this should always be pre-zeroed,
828	 * since it was just mmap()ed, but let's be sure.
829	 */
830	memset(arenas, 0, sizeof(arena_t *) * narenas_total);
831	/* Copy the pointer to the one arena that was already initialized. */
832	arenas[0] = init_arenas[0];
833
834	malloc_initialized = true;
835	malloc_mutex_unlock(&init_lock);
836
837	return (false);
838}
839
840/*
841 * End initialization functions.
842 */
843/******************************************************************************/
844/*
845 * Begin malloc(3)-compatible functions.
846 */
847
848static void *
849imalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt)
850{
851	void *p;
852
853	if (cnt == NULL)
854		return (NULL);
855	if (usize <= SMALL_MAXCLASS) {
856		p = imalloc(SMALL_MAXCLASS+1);
857		if (p == NULL)
858			return (NULL);
859		arena_prof_promoted(p, usize);
860	} else
861		p = imalloc(usize);
862
863	return (p);
864}
865
866JEMALLOC_ALWAYS_INLINE_C void *
867imalloc_prof(size_t usize, prof_thr_cnt_t *cnt)
868{
869	void *p;
870
871	if ((uintptr_t)cnt != (uintptr_t)1U)
872		p = imalloc_prof_sample(usize, cnt);
873	else
874		p = imalloc(usize);
875	if (p == NULL)
876		return (NULL);
877	prof_malloc(p, usize, cnt);
878
879	return (p);
880}
881
882/*
883 * MALLOC_BODY() is a macro rather than a function because its contents are in
884 * the fast path, but inlining would cause reliability issues when determining
885 * how many frames to discard from heap profiling backtraces.
886 */
887#define	MALLOC_BODY(ret, size, usize) do {				\
888	if (malloc_init())						\
889		ret = NULL;						\
890	else {								\
891		if (config_prof && opt_prof) {				\
892			prof_thr_cnt_t *cnt;				\
893									\
894			usize = s2u(size);				\
895			/*						\
896			 * Call PROF_ALLOC_PREP() here rather than in	\
897			 * imalloc_prof() so that imalloc_prof() can be	\
898			 * inlined without introducing uncertainty	\
899			 * about the number of backtrace frames to	\
900			 * ignore.  imalloc_prof() is in the fast path	\
901			 * when heap profiling is enabled, so inlining	\
902			 * is critical to performance.  (For		\
903			 * consistency all callers of PROF_ALLOC_PREP()	\
904			 * are structured similarly, even though e.g.	\
905			 * realloc() isn't called enough for inlining	\
906			 * to be critical.)				\
907			 */						\
908			PROF_ALLOC_PREP(1, usize, cnt);			\
909			ret = imalloc_prof(usize, cnt);			\
910		} else {						\
911			if (config_stats || (config_valgrind &&		\
912			    in_valgrind))				\
913				usize = s2u(size);			\
914			ret = imalloc(size);				\
915		}							\
916	}								\
917} while (0)
918
919void *
920je_malloc(size_t size)
921{
922	void *ret;
923	size_t usize JEMALLOC_CC_SILENCE_INIT(0);
924
925	if (size == 0)
926		size = 1;
927
928	MALLOC_BODY(ret, size, usize);
929
930	if (ret == NULL) {
931		if (config_xmalloc && opt_xmalloc) {
932			malloc_write("<jemalloc>: Error in malloc(): "
933			    "out of memory\n");
934			abort();
935		}
936		set_errno(ENOMEM);
937	}
938	if (config_stats && ret != NULL) {
939		assert(usize == isalloc(ret, config_prof));
940		thread_allocated_tsd_get()->allocated += usize;
941	}
942	UTRACE(0, size, ret);
943	JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, false);
944	return (ret);
945}
946
947static void *
948imemalign_prof_sample(size_t alignment, size_t usize, prof_thr_cnt_t *cnt)
949{
950	void *p;
951
952	if (cnt == NULL)
953		return (NULL);
954	if (usize <= SMALL_MAXCLASS) {
955		assert(sa2u(SMALL_MAXCLASS+1, alignment) != 0);
956		p = ipalloc(sa2u(SMALL_MAXCLASS+1, alignment), alignment,
957		    false);
958		if (p == NULL)
959			return (NULL);
960		arena_prof_promoted(p, usize);
961	} else
962		p = ipalloc(usize, alignment, false);
963
964	return (p);
965}
966
967JEMALLOC_ALWAYS_INLINE_C void *
968imemalign_prof(size_t alignment, size_t usize, prof_thr_cnt_t *cnt)
969{
970	void *p;
971
972	if ((uintptr_t)cnt != (uintptr_t)1U)
973		p = imemalign_prof_sample(alignment, usize, cnt);
974	else
975		p = ipalloc(usize, alignment, false);
976	if (p == NULL)
977		return (NULL);
978	prof_malloc(p, usize, cnt);
979
980	return (p);
981}
982
983JEMALLOC_ATTR(nonnull(1))
984#ifdef JEMALLOC_PROF
985/*
986 * Avoid any uncertainty as to how many backtrace frames to ignore in
987 * PROF_ALLOC_PREP().
988 */
989JEMALLOC_NOINLINE
990#endif
991static int
992imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment)
993{
994	int ret;
995	size_t usize;
996	void *result;
997
998	assert(min_alignment != 0);
999
1000	if (malloc_init()) {
1001		result = NULL;
1002		goto label_oom;
1003	} else {
1004		if (size == 0)
1005			size = 1;
1006
1007		/* Make sure that alignment is a large enough power of 2. */
1008		if (((alignment - 1) & alignment) != 0
1009		    || (alignment < min_alignment)) {
1010			if (config_xmalloc && opt_xmalloc) {
1011				malloc_write("<jemalloc>: Error allocating "
1012				    "aligned memory: invalid alignment\n");
1013				abort();
1014			}
1015			result = NULL;
1016			ret = EINVAL;
1017			goto label_return;
1018		}
1019
1020		usize = sa2u(size, alignment);
1021		if (usize == 0) {
1022			result = NULL;
1023			goto label_oom;
1024		}
1025
1026		if (config_prof && opt_prof) {
1027			prof_thr_cnt_t *cnt;
1028
1029			PROF_ALLOC_PREP(2, usize, cnt);
1030			result = imemalign_prof(alignment, usize, cnt);
1031		} else
1032			result = ipalloc(usize, alignment, false);
1033		if (result == NULL)
1034			goto label_oom;
1035	}
1036
1037	*memptr = result;
1038	ret = 0;
1039label_return:
1040	if (config_stats && result != NULL) {
1041		assert(usize == isalloc(result, config_prof));
1042		thread_allocated_tsd_get()->allocated += usize;
1043	}
1044	UTRACE(0, size, result);
1045	return (ret);
1046label_oom:
1047	assert(result == NULL);
1048	if (config_xmalloc && opt_xmalloc) {
1049		malloc_write("<jemalloc>: Error allocating aligned memory: "
1050		    "out of memory\n");
1051		abort();
1052	}
1053	ret = ENOMEM;
1054	goto label_return;
1055}
1056
1057int
1058je_posix_memalign(void **memptr, size_t alignment, size_t size)
1059{
1060	int ret = imemalign(memptr, alignment, size, sizeof(void *));
1061	JEMALLOC_VALGRIND_MALLOC(ret == 0, *memptr, isalloc(*memptr,
1062	    config_prof), false);
1063	return (ret);
1064}
1065
1066void *
1067je_aligned_alloc(size_t alignment, size_t size)
1068{
1069	void *ret;
1070	int err;
1071
1072	if ((err = imemalign(&ret, alignment, size, 1)) != 0) {
1073		ret = NULL;
1074		set_errno(err);
1075	}
1076	JEMALLOC_VALGRIND_MALLOC(err == 0, ret, isalloc(ret, config_prof),
1077	    false);
1078	return (ret);
1079}
1080
1081static void *
1082icalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt)
1083{
1084	void *p;
1085
1086	if (cnt == NULL)
1087		return (NULL);
1088	if (usize <= SMALL_MAXCLASS) {
1089		p = icalloc(SMALL_MAXCLASS+1);
1090		if (p == NULL)
1091			return (NULL);
1092		arena_prof_promoted(p, usize);
1093	} else
1094		p = icalloc(usize);
1095
1096	return (p);
1097}
1098
1099JEMALLOC_ALWAYS_INLINE_C void *
1100icalloc_prof(size_t usize, prof_thr_cnt_t *cnt)
1101{
1102	void *p;
1103
1104	if ((uintptr_t)cnt != (uintptr_t)1U)
1105		p = icalloc_prof_sample(usize, cnt);
1106	else
1107		p = icalloc(usize);
1108	if (p == NULL)
1109		return (NULL);
1110	prof_malloc(p, usize, cnt);
1111
1112	return (p);
1113}
1114
1115void *
1116je_calloc(size_t num, size_t size)
1117{
1118	void *ret;
1119	size_t num_size;
1120	size_t usize JEMALLOC_CC_SILENCE_INIT(0);
1121
1122	if (malloc_init()) {
1123		num_size = 0;
1124		ret = NULL;
1125		goto label_return;
1126	}
1127
1128	num_size = num * size;
1129	if (num_size == 0) {
1130		if (num == 0 || size == 0)
1131			num_size = 1;
1132		else {
1133			ret = NULL;
1134			goto label_return;
1135		}
1136	/*
1137	 * Try to avoid division here.  We know that it isn't possible to
1138	 * overflow during multiplication if neither operand uses any of the
1139	 * most significant half of the bits in a size_t.
1140	 */
1141	} else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2)))
1142	    && (num_size / size != num)) {
1143		/* size_t overflow. */
1144		ret = NULL;
1145		goto label_return;
1146	}
1147
1148	if (config_prof && opt_prof) {
1149		prof_thr_cnt_t *cnt;
1150
1151		usize = s2u(num_size);
1152		PROF_ALLOC_PREP(1, usize, cnt);
1153		ret = icalloc_prof(usize, cnt);
1154	} else {
1155		if (config_stats || (config_valgrind && in_valgrind))
1156			usize = s2u(num_size);
1157		ret = icalloc(num_size);
1158	}
1159
1160label_return:
1161	if (ret == NULL) {
1162		if (config_xmalloc && opt_xmalloc) {
1163			malloc_write("<jemalloc>: Error in calloc(): out of "
1164			    "memory\n");
1165			abort();
1166		}
1167		set_errno(ENOMEM);
1168	}
1169	if (config_stats && ret != NULL) {
1170		assert(usize == isalloc(ret, config_prof));
1171		thread_allocated_tsd_get()->allocated += usize;
1172	}
1173	UTRACE(0, num_size, ret);
1174	JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, true);
1175	return (ret);
1176}
1177
1178static void *
1179irealloc_prof_sample(void *oldptr, size_t usize, prof_thr_cnt_t *cnt)
1180{
1181	void *p;
1182
1183	if (cnt == NULL)
1184		return (NULL);
1185	if (usize <= SMALL_MAXCLASS) {
1186		p = iralloc(oldptr, SMALL_MAXCLASS+1, 0, 0, false);
1187		if (p == NULL)
1188			return (NULL);
1189		arena_prof_promoted(p, usize);
1190	} else
1191		p = iralloc(oldptr, usize, 0, 0, false);
1192
1193	return (p);
1194}
1195
1196JEMALLOC_ALWAYS_INLINE_C void *
1197irealloc_prof(void *oldptr, size_t old_usize, size_t usize, prof_thr_cnt_t *cnt)
1198{
1199	void *p;
1200	prof_ctx_t *old_ctx;
1201
1202	old_ctx = prof_ctx_get(oldptr);
1203	if ((uintptr_t)cnt != (uintptr_t)1U)
1204		p = irealloc_prof_sample(oldptr, usize, cnt);
1205	else
1206		p = iralloc(oldptr, usize, 0, 0, false);
1207	if (p == NULL)
1208		return (NULL);
1209	prof_realloc(p, usize, cnt, old_usize, old_ctx);
1210
1211	return (p);
1212}
1213
1214JEMALLOC_INLINE_C void
1215ifree(void *ptr)
1216{
1217	size_t usize;
1218	UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
1219
1220	assert(ptr != NULL);
1221	assert(malloc_initialized || IS_INITIALIZER);
1222
1223	if (config_prof && opt_prof) {
1224		usize = isalloc(ptr, config_prof);
1225		prof_free(ptr, usize);
1226	} else if (config_stats || config_valgrind)
1227		usize = isalloc(ptr, config_prof);
1228	if (config_stats)
1229		thread_allocated_tsd_get()->deallocated += usize;
1230	if (config_valgrind && in_valgrind)
1231		rzsize = p2rz(ptr);
1232	iqalloc(ptr);
1233	JEMALLOC_VALGRIND_FREE(ptr, rzsize);
1234}
1235
1236void *
1237je_realloc(void *ptr, size_t size)
1238{
1239	void *ret;
1240	size_t usize JEMALLOC_CC_SILENCE_INIT(0);
1241	size_t old_usize = 0;
1242	UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
1243
1244	if (size == 0) {
1245		if (ptr != NULL) {
1246			/* realloc(ptr, 0) is equivalent to free(ptr). */
1247			UTRACE(ptr, 0, 0);
1248			ifree(ptr);
1249			return (NULL);
1250		}
1251		size = 1;
1252	}
1253
1254	if (ptr != NULL) {
1255		assert(malloc_initialized || IS_INITIALIZER);
1256		malloc_thread_init();
1257
1258		if ((config_prof && opt_prof) || config_stats ||
1259		    (config_valgrind && in_valgrind))
1260			old_usize = isalloc(ptr, config_prof);
1261		if (config_valgrind && in_valgrind)
1262			old_rzsize = config_prof ? p2rz(ptr) : u2rz(old_usize);
1263
1264		if (config_prof && opt_prof) {
1265			prof_thr_cnt_t *cnt;
1266
1267			usize = s2u(size);
1268			PROF_ALLOC_PREP(1, usize, cnt);
1269			ret = irealloc_prof(ptr, old_usize, usize, cnt);
1270		} else {
1271			if (config_stats || (config_valgrind && in_valgrind))
1272				usize = s2u(size);
1273			ret = iralloc(ptr, size, 0, 0, false);
1274		}
1275	} else {
1276		/* realloc(NULL, size) is equivalent to malloc(size). */
1277		MALLOC_BODY(ret, size, usize);
1278	}
1279
1280	if (ret == NULL) {
1281		if (config_xmalloc && opt_xmalloc) {
1282			malloc_write("<jemalloc>: Error in realloc(): "
1283			    "out of memory\n");
1284			abort();
1285		}
1286		set_errno(ENOMEM);
1287	}
1288	if (config_stats && ret != NULL) {
1289		thread_allocated_t *ta;
1290		assert(usize == isalloc(ret, config_prof));
1291		ta = thread_allocated_tsd_get();
1292		ta->allocated += usize;
1293		ta->deallocated += old_usize;
1294	}
1295	UTRACE(ptr, size, ret);
1296	JEMALLOC_VALGRIND_REALLOC(ret, usize, ptr, old_usize, old_rzsize,
1297	    false);
1298	return (ret);
1299}
1300
1301void
1302je_free(void *ptr)
1303{
1304
1305	UTRACE(ptr, 0, 0);
1306	if (ptr != NULL)
1307		ifree(ptr);
1308}
1309
1310/*
1311 * End malloc(3)-compatible functions.
1312 */
1313/******************************************************************************/
1314/*
1315 * Begin non-standard override functions.
1316 */
1317
1318#ifdef JEMALLOC_OVERRIDE_MEMALIGN
1319void *
1320je_memalign(size_t alignment, size_t size)
1321{
1322	void *ret JEMALLOC_CC_SILENCE_INIT(NULL);
1323	imemalign(&ret, alignment, size, 1);
1324	JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false);
1325	return (ret);
1326}
1327#endif
1328
1329#ifdef JEMALLOC_OVERRIDE_VALLOC
1330void *
1331je_valloc(size_t size)
1332{
1333	void *ret JEMALLOC_CC_SILENCE_INIT(NULL);
1334	imemalign(&ret, PAGE, size, 1);
1335	JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false);
1336	return (ret);
1337}
1338#endif
1339
1340/*
1341 * is_malloc(je_malloc) is some macro magic to detect if jemalloc_defs.h has
1342 * #define je_malloc malloc
1343 */
1344#define	malloc_is_malloc 1
1345#define	is_malloc_(a) malloc_is_ ## a
1346#define	is_malloc(a) is_malloc_(a)
1347
1348#if ((is_malloc(je_malloc) == 1) && defined(__GLIBC__) && !defined(__UCLIBC__))
1349/*
1350 * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible
1351 * to inconsistently reference libc's malloc(3)-compatible functions
1352 * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541).
1353 *
1354 * These definitions interpose hooks in glibc.  The functions are actually
1355 * passed an extra argument for the caller return address, which will be
1356 * ignored.
1357 */
1358JEMALLOC_EXPORT void (* __free_hook)(void *ptr) = je_free;
1359JEMALLOC_EXPORT void *(* __malloc_hook)(size_t size) = je_malloc;
1360JEMALLOC_EXPORT void *(* __realloc_hook)(void *ptr, size_t size) = je_realloc;
1361JEMALLOC_EXPORT void *(* __memalign_hook)(size_t alignment, size_t size) =
1362    je_memalign;
1363#endif
1364
1365/*
1366 * End non-standard override functions.
1367 */
1368/******************************************************************************/
1369/*
1370 * Begin non-standard functions.
1371 */
1372
1373JEMALLOC_ALWAYS_INLINE_C void *
1374imallocx(size_t usize, size_t alignment, bool zero, bool try_tcache,
1375    arena_t *arena)
1376{
1377
1378	assert(usize == ((alignment == 0) ? s2u(usize) : sa2u(usize,
1379	    alignment)));
1380
1381	if (alignment != 0)
1382		return (ipalloct(usize, alignment, zero, try_tcache, arena));
1383	else if (zero)
1384		return (icalloct(usize, try_tcache, arena));
1385	else
1386		return (imalloct(usize, try_tcache, arena));
1387}
1388
1389static void *
1390imallocx_prof_sample(size_t usize, size_t alignment, bool zero, bool try_tcache,
1391    arena_t *arena, prof_thr_cnt_t *cnt)
1392{
1393	void *p;
1394
1395	if (cnt == NULL)
1396		return (NULL);
1397	if (usize <= SMALL_MAXCLASS) {
1398		size_t usize_promoted = (alignment == 0) ?
1399		    s2u(SMALL_MAXCLASS+1) : sa2u(SMALL_MAXCLASS+1, alignment);
1400		assert(usize_promoted != 0);
1401		p = imallocx(usize_promoted, alignment, zero, try_tcache,
1402		    arena);
1403		if (p == NULL)
1404			return (NULL);
1405		arena_prof_promoted(p, usize);
1406	} else
1407		p = imallocx(usize, alignment, zero, try_tcache, arena);
1408
1409	return (p);
1410}
1411
1412JEMALLOC_ALWAYS_INLINE_C void *
1413imallocx_prof(size_t usize, size_t alignment, bool zero, bool try_tcache,
1414    arena_t *arena, prof_thr_cnt_t *cnt)
1415{
1416	void *p;
1417
1418	if ((uintptr_t)cnt != (uintptr_t)1U) {
1419		p = imallocx_prof_sample(usize, alignment, zero, try_tcache,
1420		    arena, cnt);
1421	} else
1422		p = imallocx(usize, alignment, zero, try_tcache, arena);
1423	if (p == NULL)
1424		return (NULL);
1425	prof_malloc(p, usize, cnt);
1426
1427	return (p);
1428}
1429
1430void *
1431je_mallocx(size_t size, int flags)
1432{
1433	void *p;
1434	size_t usize;
1435	size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK)
1436	    & (SIZE_T_MAX-1));
1437	bool zero = flags & MALLOCX_ZERO;
1438	unsigned arena_ind = ((unsigned)(flags >> 8)) - 1;
1439	arena_t *arena;
1440	bool try_tcache;
1441
1442	assert(size != 0);
1443
1444	if (malloc_init())
1445		goto label_oom;
1446
1447	if (arena_ind != UINT_MAX) {
1448		arena = arenas[arena_ind];
1449		try_tcache = false;
1450	} else {
1451		arena = NULL;
1452		try_tcache = true;
1453	}
1454
1455	usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
1456	assert(usize != 0);
1457
1458	if (config_prof && opt_prof) {
1459		prof_thr_cnt_t *cnt;
1460
1461		PROF_ALLOC_PREP(1, usize, cnt);
1462		p = imallocx_prof(usize, alignment, zero, try_tcache, arena,
1463		    cnt);
1464	} else
1465		p = imallocx(usize, alignment, zero, try_tcache, arena);
1466	if (p == NULL)
1467		goto label_oom;
1468
1469	if (config_stats) {
1470		assert(usize == isalloc(p, config_prof));
1471		thread_allocated_tsd_get()->allocated += usize;
1472	}
1473	UTRACE(0, size, p);
1474	JEMALLOC_VALGRIND_MALLOC(true, p, usize, zero);
1475	return (p);
1476label_oom:
1477	if (config_xmalloc && opt_xmalloc) {
1478		malloc_write("<jemalloc>: Error in mallocx(): out of memory\n");
1479		abort();
1480	}
1481	UTRACE(0, size, 0);
1482	return (NULL);
1483}
1484
1485static void *
1486irallocx_prof_sample(void *oldptr, size_t size, size_t alignment, size_t usize,
1487    bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena,
1488    prof_thr_cnt_t *cnt)
1489{
1490	void *p;
1491
1492	if (cnt == NULL)
1493		return (NULL);
1494	if (usize <= SMALL_MAXCLASS) {
1495		p = iralloct(oldptr, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >=
1496		    size) ? 0 : size - (SMALL_MAXCLASS+1), alignment, zero,
1497		    try_tcache_alloc, try_tcache_dalloc, arena);
1498		if (p == NULL)
1499			return (NULL);
1500		arena_prof_promoted(p, usize);
1501	} else {
1502		p = iralloct(oldptr, size, 0, alignment, zero,
1503		    try_tcache_alloc, try_tcache_dalloc, arena);
1504	}
1505
1506	return (p);
1507}
1508
1509JEMALLOC_ALWAYS_INLINE_C void *
1510irallocx_prof(void *oldptr, size_t old_usize, size_t size, size_t alignment,
1511    size_t *usize, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc,
1512    arena_t *arena, prof_thr_cnt_t *cnt)
1513{
1514	void *p;
1515	prof_ctx_t *old_ctx;
1516
1517	old_ctx = prof_ctx_get(oldptr);
1518	if ((uintptr_t)cnt != (uintptr_t)1U)
1519		p = irallocx_prof_sample(oldptr, size, alignment, *usize, zero,
1520		    try_tcache_alloc, try_tcache_dalloc, arena, cnt);
1521	else {
1522		p = iralloct(oldptr, size, 0, alignment, zero,
1523		    try_tcache_alloc, try_tcache_dalloc, arena);
1524	}
1525	if (p == NULL)
1526		return (NULL);
1527
1528	if (p == oldptr && alignment != 0) {
1529		/*
1530		 * The allocation did not move, so it is possible that the size
1531		 * class is smaller than would guarantee the requested
1532		 * alignment, and that the alignment constraint was
1533		 * serendipitously satisfied.  Additionally, old_usize may not
1534		 * be the same as the current usize because of in-place large
1535		 * reallocation.  Therefore, query the actual value of usize.
1536		 */
1537		*usize = isalloc(p, config_prof);
1538	}
1539	prof_realloc(p, *usize, cnt, old_usize, old_ctx);
1540
1541	return (p);
1542}
1543
1544void *
1545je_rallocx(void *ptr, size_t size, int flags)
1546{
1547	void *p;
1548	size_t usize, old_usize;
1549	UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
1550	size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK)
1551	    & (SIZE_T_MAX-1));
1552	bool zero = flags & MALLOCX_ZERO;
1553	unsigned arena_ind = ((unsigned)(flags >> 8)) - 1;
1554	bool try_tcache_alloc, try_tcache_dalloc;
1555	arena_t *arena;
1556
1557	assert(ptr != NULL);
1558	assert(size != 0);
1559	assert(malloc_initialized || IS_INITIALIZER);
1560	malloc_thread_init();
1561
1562	if (arena_ind != UINT_MAX) {
1563		arena_chunk_t *chunk;
1564		try_tcache_alloc = false;
1565		chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
1566		try_tcache_dalloc = (chunk == ptr || chunk->arena !=
1567		    arenas[arena_ind]);
1568		arena = arenas[arena_ind];
1569	} else {
1570		try_tcache_alloc = true;
1571		try_tcache_dalloc = true;
1572		arena = NULL;
1573	}
1574
1575	if ((config_prof && opt_prof) || config_stats ||
1576	    (config_valgrind && in_valgrind))
1577		old_usize = isalloc(ptr, config_prof);
1578	if (config_valgrind && in_valgrind)
1579		old_rzsize = u2rz(old_usize);
1580
1581	if (config_prof && opt_prof) {
1582		prof_thr_cnt_t *cnt;
1583
1584		usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
1585		assert(usize != 0);
1586		PROF_ALLOC_PREP(1, usize, cnt);
1587		p = irallocx_prof(ptr, old_usize, size, alignment, &usize, zero,
1588		    try_tcache_alloc, try_tcache_dalloc, arena, cnt);
1589		if (p == NULL)
1590			goto label_oom;
1591	} else {
1592		p = iralloct(ptr, size, 0, alignment, zero, try_tcache_alloc,
1593		    try_tcache_dalloc, arena);
1594		if (p == NULL)
1595			goto label_oom;
1596		if (config_stats || (config_valgrind && in_valgrind))
1597			usize = isalloc(p, config_prof);
1598	}
1599
1600	if (config_stats) {
1601		thread_allocated_t *ta;
1602		ta = thread_allocated_tsd_get();
1603		ta->allocated += usize;
1604		ta->deallocated += old_usize;
1605	}
1606	UTRACE(ptr, size, p);
1607	JEMALLOC_VALGRIND_REALLOC(p, usize, ptr, old_usize, old_rzsize, zero);
1608	return (p);
1609label_oom:
1610	if (config_xmalloc && opt_xmalloc) {
1611		malloc_write("<jemalloc>: Error in rallocx(): out of memory\n");
1612		abort();
1613	}
1614	UTRACE(ptr, size, 0);
1615	return (NULL);
1616}
1617
1618JEMALLOC_ALWAYS_INLINE_C size_t
1619ixallocx_helper(void *ptr, size_t old_usize, size_t size, size_t extra,
1620    size_t alignment, bool zero, arena_t *arena)
1621{
1622	size_t usize;
1623
1624	if (ixalloc(ptr, size, extra, alignment, zero))
1625		return (old_usize);
1626	usize = isalloc(ptr, config_prof);
1627
1628	return (usize);
1629}
1630
1631static size_t
1632ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra,
1633    size_t alignment, size_t max_usize, bool zero, arena_t *arena,
1634    prof_thr_cnt_t *cnt)
1635{
1636	size_t usize;
1637
1638	if (cnt == NULL)
1639		return (old_usize);
1640	/* Use minimum usize to determine whether promotion may happen. */
1641	if (((alignment == 0) ? s2u(size) : sa2u(size, alignment)) <=
1642	    SMALL_MAXCLASS) {
1643		if (ixalloc(ptr, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >=
1644		    size+extra) ? 0 : size+extra - (SMALL_MAXCLASS+1),
1645		    alignment, zero))
1646			return (old_usize);
1647		usize = isalloc(ptr, config_prof);
1648		if (max_usize < PAGE)
1649			arena_prof_promoted(ptr, usize);
1650	} else {
1651		usize = ixallocx_helper(ptr, old_usize, size, extra, alignment,
1652		    zero, arena);
1653	}
1654
1655	return (usize);
1656}
1657
1658JEMALLOC_ALWAYS_INLINE_C size_t
1659ixallocx_prof(void *ptr, size_t old_usize, size_t size, size_t extra,
1660    size_t alignment, size_t max_usize, bool zero, arena_t *arena,
1661    prof_thr_cnt_t *cnt)
1662{
1663	size_t usize;
1664	prof_ctx_t *old_ctx;
1665
1666	old_ctx = prof_ctx_get(ptr);
1667	if ((uintptr_t)cnt != (uintptr_t)1U) {
1668		usize = ixallocx_prof_sample(ptr, old_usize, size, extra,
1669		    alignment, zero, max_usize, arena, cnt);
1670	} else {
1671		usize = ixallocx_helper(ptr, old_usize, size, extra, alignment,
1672		    zero, arena);
1673	}
1674	if (usize == old_usize)
1675		return (usize);
1676	prof_realloc(ptr, usize, cnt, old_usize, old_ctx);
1677
1678	return (usize);
1679}
1680
1681size_t
1682je_xallocx(void *ptr, size_t size, size_t extra, int flags)
1683{
1684	size_t usize, old_usize;
1685	UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
1686	size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK)
1687	    & (SIZE_T_MAX-1));
1688	bool zero = flags & MALLOCX_ZERO;
1689	unsigned arena_ind = ((unsigned)(flags >> 8)) - 1;
1690	arena_t *arena;
1691
1692	assert(ptr != NULL);
1693	assert(size != 0);
1694	assert(SIZE_T_MAX - size >= extra);
1695	assert(malloc_initialized || IS_INITIALIZER);
1696	malloc_thread_init();
1697
1698	if (arena_ind != UINT_MAX)
1699		arena = arenas[arena_ind];
1700	else
1701		arena = NULL;
1702
1703	old_usize = isalloc(ptr, config_prof);
1704	if (config_valgrind && in_valgrind)
1705		old_rzsize = u2rz(old_usize);
1706
1707	if (config_prof && opt_prof) {
1708		prof_thr_cnt_t *cnt;
1709		/*
1710		 * usize isn't knowable before ixalloc() returns when extra is
1711		 * non-zero.  Therefore, compute its maximum possible value and
1712		 * use that in PROF_ALLOC_PREP() to decide whether to capture a
1713		 * backtrace.  prof_realloc() will use the actual usize to
1714		 * decide whether to sample.
1715		 */
1716		size_t max_usize = (alignment == 0) ? s2u(size+extra) :
1717		    sa2u(size+extra, alignment);
1718		PROF_ALLOC_PREP(1, max_usize, cnt);
1719		usize = ixallocx_prof(ptr, old_usize, size, extra, alignment,
1720		    max_usize, zero, arena, cnt);
1721	} else {
1722		usize = ixallocx_helper(ptr, old_usize, size, extra, alignment,
1723		    zero, arena);
1724	}
1725	if (usize == old_usize)
1726		goto label_not_resized;
1727
1728	if (config_stats) {
1729		thread_allocated_t *ta;
1730		ta = thread_allocated_tsd_get();
1731		ta->allocated += usize;
1732		ta->deallocated += old_usize;
1733	}
1734	JEMALLOC_VALGRIND_REALLOC(ptr, usize, ptr, old_usize, old_rzsize, zero);
1735label_not_resized:
1736	UTRACE(ptr, size, ptr);
1737	return (usize);
1738}
1739
1740size_t
1741je_sallocx(const void *ptr, int flags)
1742{
1743	size_t usize;
1744
1745	assert(malloc_initialized || IS_INITIALIZER);
1746	malloc_thread_init();
1747
1748	if (config_ivsalloc)
1749		usize = ivsalloc(ptr, config_prof);
1750	else {
1751		assert(ptr != NULL);
1752		usize = isalloc(ptr, config_prof);
1753	}
1754
1755	return (usize);
1756}
1757
1758void
1759je_dallocx(void *ptr, int flags)
1760{
1761	size_t usize;
1762	UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
1763	unsigned arena_ind = ((unsigned)(flags >> 8)) - 1;
1764	bool try_tcache;
1765
1766	assert(ptr != NULL);
1767	assert(malloc_initialized || IS_INITIALIZER);
1768
1769	if (arena_ind != UINT_MAX) {
1770		arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
1771		try_tcache = (chunk == ptr || chunk->arena !=
1772		    arenas[arena_ind]);
1773	} else
1774		try_tcache = true;
1775
1776	UTRACE(ptr, 0, 0);
1777	if (config_stats || config_valgrind)
1778		usize = isalloc(ptr, config_prof);
1779	if (config_prof && opt_prof) {
1780		if (config_stats == false && config_valgrind == false)
1781			usize = isalloc(ptr, config_prof);
1782		prof_free(ptr, usize);
1783	}
1784	if (config_stats)
1785		thread_allocated_tsd_get()->deallocated += usize;
1786	if (config_valgrind && in_valgrind)
1787		rzsize = p2rz(ptr);
1788	iqalloct(ptr, try_tcache);
1789	JEMALLOC_VALGRIND_FREE(ptr, rzsize);
1790}
1791
1792size_t
1793je_nallocx(size_t size, int flags)
1794{
1795	size_t usize;
1796	size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK)
1797	    & (SIZE_T_MAX-1));
1798
1799	assert(size != 0);
1800
1801	if (malloc_init())
1802		return (0);
1803
1804	usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
1805	assert(usize != 0);
1806	return (usize);
1807}
1808
1809int
1810je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp,
1811    size_t newlen)
1812{
1813
1814	if (malloc_init())
1815		return (EAGAIN);
1816
1817	return (ctl_byname(name, oldp, oldlenp, newp, newlen));
1818}
1819
1820int
1821je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp)
1822{
1823
1824	if (malloc_init())
1825		return (EAGAIN);
1826
1827	return (ctl_nametomib(name, mibp, miblenp));
1828}
1829
1830int
1831je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
1832  void *newp, size_t newlen)
1833{
1834
1835	if (malloc_init())
1836		return (EAGAIN);
1837
1838	return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen));
1839}
1840
1841void
1842je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
1843    const char *opts)
1844{
1845
1846	stats_print(write_cb, cbopaque, opts);
1847}
1848
1849size_t
1850je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr)
1851{
1852	size_t ret;
1853
1854	assert(malloc_initialized || IS_INITIALIZER);
1855	malloc_thread_init();
1856
1857	if (config_ivsalloc)
1858		ret = ivsalloc(ptr, config_prof);
1859	else
1860		ret = (ptr != NULL) ? isalloc(ptr, config_prof) : 0;
1861
1862	return (ret);
1863}
1864
1865/*
1866 * End non-standard functions.
1867 */
1868/******************************************************************************/
1869/*
1870 * The following functions are used by threading libraries for protection of
1871 * malloc during fork().
1872 */
1873
1874/*
1875 * If an application creates a thread before doing any allocation in the main
1876 * thread, then calls fork(2) in the main thread followed by memory allocation
1877 * in the child process, a race can occur that results in deadlock within the
1878 * child: the main thread may have forked while the created thread had
1879 * partially initialized the allocator.  Ordinarily jemalloc prevents
1880 * fork/malloc races via the following functions it registers during
1881 * initialization using pthread_atfork(), but of course that does no good if
1882 * the allocator isn't fully initialized at fork time.  The following library
1883 * constructor is a partial solution to this problem.  It may still possible to
1884 * trigger the deadlock described above, but doing so would involve forking via
1885 * a library constructor that runs before jemalloc's runs.
1886 */
1887JEMALLOC_ATTR(constructor)
1888static void
1889jemalloc_constructor(void)
1890{
1891
1892	malloc_init();
1893}
1894
1895#ifndef JEMALLOC_MUTEX_INIT_CB
1896void
1897jemalloc_prefork(void)
1898#else
1899JEMALLOC_EXPORT void
1900_malloc_prefork(void)
1901#endif
1902{
1903	unsigned i;
1904
1905#ifdef JEMALLOC_MUTEX_INIT_CB
1906	if (malloc_initialized == false)
1907		return;
1908#endif
1909	assert(malloc_initialized);
1910
1911	/* Acquire all mutexes in a safe order. */
1912	ctl_prefork();
1913	prof_prefork();
1914	malloc_mutex_prefork(&arenas_lock);
1915	for (i = 0; i < narenas_total; i++) {
1916		if (arenas[i] != NULL)
1917			arena_prefork(arenas[i]);
1918	}
1919	chunk_prefork();
1920	base_prefork();
1921	huge_prefork();
1922}
1923
1924#ifndef JEMALLOC_MUTEX_INIT_CB
1925void
1926jemalloc_postfork_parent(void)
1927#else
1928JEMALLOC_EXPORT void
1929_malloc_postfork(void)
1930#endif
1931{
1932	unsigned i;
1933
1934#ifdef JEMALLOC_MUTEX_INIT_CB
1935	if (malloc_initialized == false)
1936		return;
1937#endif
1938	assert(malloc_initialized);
1939
1940	/* Release all mutexes, now that fork() has completed. */
1941	huge_postfork_parent();
1942	base_postfork_parent();
1943	chunk_postfork_parent();
1944	for (i = 0; i < narenas_total; i++) {
1945		if (arenas[i] != NULL)
1946			arena_postfork_parent(arenas[i]);
1947	}
1948	malloc_mutex_postfork_parent(&arenas_lock);
1949	prof_postfork_parent();
1950	ctl_postfork_parent();
1951}
1952
1953void
1954jemalloc_postfork_child(void)
1955{
1956	unsigned i;
1957
1958	assert(malloc_initialized);
1959
1960	/* Release all mutexes, now that fork() has completed. */
1961	huge_postfork_child();
1962	base_postfork_child();
1963	chunk_postfork_child();
1964	for (i = 0; i < narenas_total; i++) {
1965		if (arenas[i] != NULL)
1966			arena_postfork_child(arenas[i]);
1967	}
1968	malloc_mutex_postfork_child(&arenas_lock);
1969	prof_postfork_child();
1970	ctl_postfork_child();
1971}
1972
1973/******************************************************************************/
1974/*
1975 * The following functions are used for TLS allocation/deallocation in static
1976 * binaries on FreeBSD.  The primary difference between these and i[mcd]alloc()
1977 * is that these avoid accessing TLS variables.
1978 */
1979
1980static void *
1981a0alloc(size_t size, bool zero)
1982{
1983
1984	if (malloc_init())
1985		return (NULL);
1986
1987	if (size == 0)
1988		size = 1;
1989
1990	if (size <= arena_maxclass)
1991		return (arena_malloc(arenas[0], size, zero, false));
1992	else
1993		return (huge_malloc(size, zero, huge_dss_prec_get(arenas[0])));
1994}
1995
1996void *
1997a0malloc(size_t size)
1998{
1999
2000	return (a0alloc(size, false));
2001}
2002
2003void *
2004a0calloc(size_t num, size_t size)
2005{
2006
2007	return (a0alloc(num * size, true));
2008}
2009
2010void
2011a0free(void *ptr)
2012{
2013	arena_chunk_t *chunk;
2014
2015	if (ptr == NULL)
2016		return;
2017
2018	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
2019	if (chunk != ptr)
2020		arena_dalloc(chunk, ptr, false);
2021	else
2022		huge_dalloc(ptr, true);
2023}
2024
2025/******************************************************************************/
2026