1#define	JEMALLOC_C_
2#include "jemalloc/internal/jemalloc_internal.h"
3
4/******************************************************************************/
5/* Data. */
6
7malloc_tsd_data(, arenas, arena_t *, NULL)
8malloc_tsd_data(, thread_allocated, thread_allocated_t,
9    THREAD_ALLOCATED_INITIALIZER)
10
11/* Runtime configuration options. */
12const char	*je_malloc_conf;
13bool	opt_abort =
14#ifdef JEMALLOC_DEBUG
15    true
16#else
17    false
18#endif
19    ;
20bool	opt_junk =
21#if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
22    true
23#else
24    false
25#endif
26    ;
27size_t	opt_quarantine = ZU(0);
28bool	opt_redzone = false;
29bool	opt_utrace = false;
30bool	opt_xmalloc = false;
31bool	opt_zero = false;
32size_t	opt_narenas = 0;
33
34/* Initialized to true if the process is running inside Valgrind. */
35bool	in_valgrind;
36
37unsigned	ncpus;
38
39malloc_mutex_t		arenas_lock;
40arena_t			**arenas;
41unsigned		narenas_total;
42unsigned		narenas_auto;
43
44/* Set to true once the allocator has been initialized. */
45static bool		malloc_initialized = false;
46
47#ifdef JEMALLOC_THREADED_INIT
48/* Used to let the initializing thread recursively allocate. */
49#  define NO_INITIALIZER	((unsigned long)0)
50#  define INITIALIZER		pthread_self()
51#  define IS_INITIALIZER	(malloc_initializer == pthread_self())
52static pthread_t		malloc_initializer = NO_INITIALIZER;
53#else
54#  define NO_INITIALIZER	false
55#  define INITIALIZER		true
56#  define IS_INITIALIZER	malloc_initializer
57static bool			malloc_initializer = NO_INITIALIZER;
58#endif
59
60/* Used to avoid initialization races. */
61#ifdef _WIN32
62static malloc_mutex_t	init_lock;
63
64JEMALLOC_ATTR(constructor)
65static void WINAPI
66_init_init_lock(void)
67{
68
69	malloc_mutex_init(&init_lock);
70}
71
72#ifdef _MSC_VER
73#  pragma section(".CRT$XCU", read)
74JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used)
75static const void (WINAPI *init_init_lock)(void) = _init_init_lock;
76#endif
77
78#else
79static malloc_mutex_t	init_lock = MALLOC_MUTEX_INITIALIZER;
80#endif
81
82typedef struct {
83	void	*p;	/* Input pointer (as in realloc(p, s)). */
84	size_t	s;	/* Request size. */
85	void	*r;	/* Result pointer. */
86} malloc_utrace_t;
87
88#ifdef JEMALLOC_UTRACE
89#  define UTRACE(a, b, c) do {						\
90	if (opt_utrace) {						\
91		int utrace_serrno = errno;				\
92		malloc_utrace_t ut;					\
93		ut.p = (a);						\
94		ut.s = (b);						\
95		ut.r = (c);						\
96		utrace(&ut, sizeof(ut));				\
97		errno = utrace_serrno;					\
98	}								\
99} while (0)
100#else
101#  define UTRACE(a, b, c)
102#endif
103
104/******************************************************************************/
105/*
106 * Function prototypes for static functions that are referenced prior to
107 * definition.
108 */
109
110static bool	malloc_init_hard(void);
111
112/******************************************************************************/
113/*
114 * Begin miscellaneous support functions.
115 */
116
117/* Create a new arena and insert it into the arenas array at index ind. */
118arena_t *
119arenas_extend(unsigned ind)
120{
121	arena_t *ret;
122
123	ret = (arena_t *)base_alloc(sizeof(arena_t));
124	if (ret != NULL && arena_new(ret, ind) == false) {
125		arenas[ind] = ret;
126		return (ret);
127	}
128	/* Only reached if there is an OOM error. */
129
130	/*
131	 * OOM here is quite inconvenient to propagate, since dealing with it
132	 * would require a check for failure in the fast path.  Instead, punt
133	 * by using arenas[0].  In practice, this is an extremely unlikely
134	 * failure.
135	 */
136	malloc_write("<jemalloc>: Error initializing arena\n");
137	if (opt_abort)
138		abort();
139
140	return (arenas[0]);
141}
142
143/* Slow path, called only by choose_arena(). */
144arena_t *
145choose_arena_hard(void)
146{
147	arena_t *ret;
148
149	if (narenas_auto > 1) {
150		unsigned i, choose, first_null;
151
152		choose = 0;
153		first_null = narenas_auto;
154		malloc_mutex_lock(&arenas_lock);
155		assert(arenas[0] != NULL);
156		for (i = 1; i < narenas_auto; i++) {
157			if (arenas[i] != NULL) {
158				/*
159				 * Choose the first arena that has the lowest
160				 * number of threads assigned to it.
161				 */
162				if (arenas[i]->nthreads <
163				    arenas[choose]->nthreads)
164					choose = i;
165			} else if (first_null == narenas_auto) {
166				/*
167				 * Record the index of the first uninitialized
168				 * arena, in case all extant arenas are in use.
169				 *
170				 * NB: It is possible for there to be
171				 * discontinuities in terms of initialized
172				 * versus uninitialized arenas, due to the
173				 * "thread.arena" mallctl.
174				 */
175				first_null = i;
176			}
177		}
178
179		if (arenas[choose]->nthreads == 0
180		    || first_null == narenas_auto) {
181			/*
182			 * Use an unloaded arena, or the least loaded arena if
183			 * all arenas are already initialized.
184			 */
185			ret = arenas[choose];
186		} else {
187			/* Initialize a new arena. */
188			ret = arenas_extend(first_null);
189		}
190		ret->nthreads++;
191		malloc_mutex_unlock(&arenas_lock);
192	} else {
193		ret = arenas[0];
194		malloc_mutex_lock(&arenas_lock);
195		ret->nthreads++;
196		malloc_mutex_unlock(&arenas_lock);
197	}
198
199	arenas_tsd_set(&ret);
200
201	return (ret);
202}
203
204static void
205stats_print_atexit(void)
206{
207
208	if (config_tcache && config_stats) {
209		unsigned narenas, i;
210
211		/*
212		 * Merge stats from extant threads.  This is racy, since
213		 * individual threads do not lock when recording tcache stats
214		 * events.  As a consequence, the final stats may be slightly
215		 * out of date by the time they are reported, if other threads
216		 * continue to allocate.
217		 */
218		for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
219			arena_t *arena = arenas[i];
220			if (arena != NULL) {
221				tcache_t *tcache;
222
223				/*
224				 * tcache_stats_merge() locks bins, so if any
225				 * code is introduced that acquires both arena
226				 * and bin locks in the opposite order,
227				 * deadlocks may result.
228				 */
229				malloc_mutex_lock(&arena->lock);
230				ql_foreach(tcache, &arena->tcache_ql, link) {
231					tcache_stats_merge(tcache, arena);
232				}
233				malloc_mutex_unlock(&arena->lock);
234			}
235		}
236	}
237	je_malloc_stats_print(NULL, NULL, NULL);
238}
239
240/*
241 * End miscellaneous support functions.
242 */
243/******************************************************************************/
244/*
245 * Begin initialization functions.
246 */
247
248static unsigned
249malloc_ncpus(void)
250{
251	long result;
252
253#ifdef _WIN32
254	SYSTEM_INFO si;
255	GetSystemInfo(&si);
256	result = si.dwNumberOfProcessors;
257#else
258	result = sysconf(_SC_NPROCESSORS_ONLN);
259#endif
260	return ((result == -1) ? 1 : (unsigned)result);
261}
262
263void
264arenas_cleanup(void *arg)
265{
266	arena_t *arena = *(arena_t **)arg;
267
268	malloc_mutex_lock(&arenas_lock);
269	arena->nthreads--;
270	malloc_mutex_unlock(&arenas_lock);
271}
272
273JEMALLOC_ALWAYS_INLINE_C void
274malloc_thread_init(void)
275{
276
277	/*
278	 * TSD initialization can't be safely done as a side effect of
279	 * deallocation, because it is possible for a thread to do nothing but
280	 * deallocate its TLS data via free(), in which case writing to TLS
281	 * would cause write-after-free memory corruption.  The quarantine
282	 * facility *only* gets used as a side effect of deallocation, so make
283	 * a best effort attempt at initializing its TSD by hooking all
284	 * allocation events.
285	 */
286	if (config_fill && opt_quarantine)
287		quarantine_alloc_hook();
288}
289
290JEMALLOC_ALWAYS_INLINE_C bool
291malloc_init(void)
292{
293
294	if (malloc_initialized == false && malloc_init_hard())
295		return (true);
296	malloc_thread_init();
297
298	return (false);
299}
300
301static bool
302malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p,
303    char const **v_p, size_t *vlen_p)
304{
305	bool accept;
306	const char *opts = *opts_p;
307
308	*k_p = opts;
309
310	for (accept = false; accept == false;) {
311		switch (*opts) {
312		case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
313		case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
314		case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
315		case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
316		case 'Y': case 'Z':
317		case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
318		case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
319		case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
320		case 's': case 't': case 'u': case 'v': case 'w': case 'x':
321		case 'y': case 'z':
322		case '0': case '1': case '2': case '3': case '4': case '5':
323		case '6': case '7': case '8': case '9':
324		case '_':
325			opts++;
326			break;
327		case ':':
328			opts++;
329			*klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p;
330			*v_p = opts;
331			accept = true;
332			break;
333		case '\0':
334			if (opts != *opts_p) {
335				malloc_write("<jemalloc>: Conf string ends "
336				    "with key\n");
337			}
338			return (true);
339		default:
340			malloc_write("<jemalloc>: Malformed conf string\n");
341			return (true);
342		}
343	}
344
345	for (accept = false; accept == false;) {
346		switch (*opts) {
347		case ',':
348			opts++;
349			/*
350			 * Look ahead one character here, because the next time
351			 * this function is called, it will assume that end of
352			 * input has been cleanly reached if no input remains,
353			 * but we have optimistically already consumed the
354			 * comma if one exists.
355			 */
356			if (*opts == '\0') {
357				malloc_write("<jemalloc>: Conf string ends "
358				    "with comma\n");
359			}
360			*vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p;
361			accept = true;
362			break;
363		case '\0':
364			*vlen_p = (uintptr_t)opts - (uintptr_t)*v_p;
365			accept = true;
366			break;
367		default:
368			opts++;
369			break;
370		}
371	}
372
373	*opts_p = opts;
374	return (false);
375}
376
377static void
378malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v,
379    size_t vlen)
380{
381
382	malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k,
383	    (int)vlen, v);
384}
385
386static void
387malloc_conf_init(void)
388{
389	unsigned i;
390	char buf[PATH_MAX + 1];
391	const char *opts, *k, *v;
392	size_t klen, vlen;
393
394	/*
395	 * Automatically configure valgrind before processing options.  The
396	 * valgrind option remains in jemalloc 3.x for compatibility reasons.
397	 */
398	if (config_valgrind) {
399		in_valgrind = (RUNNING_ON_VALGRIND != 0) ? true : false;
400		if (config_fill && in_valgrind) {
401			opt_junk = false;
402			assert(opt_zero == false);
403			opt_quarantine = JEMALLOC_VALGRIND_QUARANTINE_DEFAULT;
404			opt_redzone = true;
405		}
406		if (config_tcache && in_valgrind)
407			opt_tcache = false;
408	}
409
410#if defined(__ANDROID__)
411	/* Android only supports compiled options. */
412	for (i = 0; i < 1; i++) {
413#else
414	for (i = 0; i < 3; i++) {
415#endif
416		/* Get runtime configuration. */
417		switch (i) {
418		case 0:
419			if (je_malloc_conf != NULL) {
420				/*
421				 * Use options that were compiled into the
422				 * program.
423				 */
424				opts = je_malloc_conf;
425			} else {
426				/* No configuration specified. */
427				buf[0] = '\0';
428				opts = buf;
429			}
430			break;
431		case 1: {
432			int linklen = 0;
433#ifndef _WIN32
434			int saved_errno = errno;
435			const char *linkname =
436#  ifdef JEMALLOC_PREFIX
437			    "/etc/"JEMALLOC_PREFIX"malloc.conf"
438#  else
439			    "/etc/malloc.conf"
440#  endif
441			    ;
442
443			/*
444			 * Try to use the contents of the "/etc/malloc.conf"
445			 * symbolic link's name.
446			 */
447			linklen = readlink(linkname, buf, sizeof(buf) - 1);
448			if (linklen == -1) {
449				/* No configuration specified. */
450				linklen = 0;
451				/* restore errno */
452				set_errno(saved_errno);
453			}
454#endif
455			buf[linklen] = '\0';
456			opts = buf;
457			break;
458		} case 2: {
459			const char *envname =
460#ifdef JEMALLOC_PREFIX
461			    JEMALLOC_CPREFIX"MALLOC_CONF"
462#else
463			    "MALLOC_CONF"
464#endif
465			    ;
466
467			if ((opts = getenv(envname)) != NULL) {
468				/*
469				 * Do nothing; opts is already initialized to
470				 * the value of the MALLOC_CONF environment
471				 * variable.
472				 */
473			} else {
474				/* No configuration specified. */
475				buf[0] = '\0';
476				opts = buf;
477			}
478			break;
479		} default:
480			not_reached();
481			buf[0] = '\0';
482			opts = buf;
483		}
484
485		while (*opts != '\0' && malloc_conf_next(&opts, &k, &klen, &v,
486		    &vlen) == false) {
487#define	CONF_MATCH(n)							\
488	(sizeof(n)-1 == klen && strncmp(n, k, klen) == 0)
489#define	CONF_HANDLE_BOOL(o, n, cont)					\
490			if (CONF_MATCH(n)) {				\
491				if (strncmp("true", v, vlen) == 0 &&	\
492				    vlen == sizeof("true")-1)		\
493					o = true;			\
494				else if (strncmp("false", v, vlen) ==	\
495				    0 && vlen == sizeof("false")-1)	\
496					o = false;			\
497				else {					\
498					malloc_conf_error(		\
499					    "Invalid conf value",	\
500					    k, klen, v, vlen);		\
501				}					\
502				if (cont)				\
503					continue;			\
504			}
505#define	CONF_HANDLE_SIZE_T(o, n, min, max, clip)			\
506			if (CONF_MATCH(n)) {				\
507				uintmax_t um;				\
508				char *end;				\
509									\
510				set_errno(0);				\
511				um = malloc_strtoumax(v, &end, 0);	\
512				if (get_errno() != 0 || (uintptr_t)end -\
513				    (uintptr_t)v != vlen) {		\
514					malloc_conf_error(		\
515					    "Invalid conf value",	\
516					    k, klen, v, vlen);		\
517				} else if (clip) {			\
518					if (min != 0 && um < min)	\
519						o = min;		\
520					else if (um > max)		\
521						o = max;		\
522					else				\
523						o = um;			\
524				} else {				\
525					if ((min != 0 && um < min) ||	\
526					    um > max) {			\
527						malloc_conf_error(	\
528						    "Out-of-range "	\
529						    "conf value",	\
530						    k, klen, v, vlen);	\
531					} else				\
532						o = um;			\
533				}					\
534				continue;				\
535			}
536#define	CONF_HANDLE_SSIZE_T(o, n, min, max)				\
537			if (CONF_MATCH(n)) {				\
538				long l;					\
539				char *end;				\
540									\
541				set_errno(0);				\
542				l = strtol(v, &end, 0);			\
543				if (get_errno() != 0 || (uintptr_t)end -\
544				    (uintptr_t)v != vlen) {		\
545					malloc_conf_error(		\
546					    "Invalid conf value",	\
547					    k, klen, v, vlen);		\
548				} else if (l < (ssize_t)min || l >	\
549				    (ssize_t)max) {			\
550					malloc_conf_error(		\
551					    "Out-of-range conf value",	\
552					    k, klen, v, vlen);		\
553				} else					\
554					o = l;				\
555				continue;				\
556			}
557#define	CONF_HANDLE_CHAR_P(o, n, d)					\
558			if (CONF_MATCH(n)) {				\
559				size_t cpylen = (vlen <=		\
560				    sizeof(o)-1) ? vlen :		\
561				    sizeof(o)-1;			\
562				strncpy(o, v, cpylen);			\
563				o[cpylen] = '\0';			\
564				continue;				\
565			}
566
567			CONF_HANDLE_BOOL(opt_abort, "abort", true)
568			/*
569			 * Chunks always require at least one header page, plus
570			 * one data page in the absence of redzones, or three
571			 * pages in the presence of redzones.  In order to
572			 * simplify options processing, fix the limit based on
573			 * config_fill.
574			 */
575			CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE +
576			    (config_fill ? 2 : 1), (sizeof(size_t) << 3) - 1,
577			    true)
578			if (strncmp("dss", k, klen) == 0) {
579				int i;
580				bool match = false;
581				for (i = 0; i < dss_prec_limit; i++) {
582					if (strncmp(dss_prec_names[i], v, vlen)
583					    == 0) {
584						if (chunk_dss_prec_set(i)) {
585							malloc_conf_error(
586							    "Error setting dss",
587							    k, klen, v, vlen);
588						} else {
589							opt_dss =
590							    dss_prec_names[i];
591							match = true;
592							break;
593						}
594					}
595				}
596				if (match == false) {
597					malloc_conf_error("Invalid conf value",
598					    k, klen, v, vlen);
599				}
600				continue;
601			}
602			CONF_HANDLE_SIZE_T(opt_narenas, "narenas", 1,
603			    SIZE_T_MAX, false)
604			CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult",
605			    -1, (sizeof(size_t) << 3) - 1)
606			CONF_HANDLE_BOOL(opt_stats_print, "stats_print", true)
607			if (config_fill) {
608				CONF_HANDLE_BOOL(opt_junk, "junk", true)
609				CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine",
610				    0, SIZE_T_MAX, false)
611				CONF_HANDLE_BOOL(opt_redzone, "redzone", true)
612				CONF_HANDLE_BOOL(opt_zero, "zero", true)
613			}
614			if (config_utrace) {
615				CONF_HANDLE_BOOL(opt_utrace, "utrace", true)
616			}
617			if (config_xmalloc) {
618				CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc", true)
619			}
620			if (config_tcache) {
621				CONF_HANDLE_BOOL(opt_tcache, "tcache",
622				    !config_valgrind || !in_valgrind)
623				if (CONF_MATCH("tcache")) {
624					assert(config_valgrind && in_valgrind);
625					if (opt_tcache) {
626						opt_tcache = false;
627						malloc_conf_error(
628						"tcache cannot be enabled "
629						"while running inside Valgrind",
630						k, klen, v, vlen);
631					}
632					continue;
633				}
634				CONF_HANDLE_SSIZE_T(opt_lg_tcache_max,
635				    "lg_tcache_max", -1,
636				    (sizeof(size_t) << 3) - 1)
637			}
638			if (config_prof) {
639				CONF_HANDLE_BOOL(opt_prof, "prof", true)
640				CONF_HANDLE_CHAR_P(opt_prof_prefix,
641				    "prof_prefix", "jeprof")
642				CONF_HANDLE_BOOL(opt_prof_active, "prof_active",
643				    true)
644				CONF_HANDLE_SSIZE_T(opt_lg_prof_sample,
645				    "lg_prof_sample", 0,
646				    (sizeof(uint64_t) << 3) - 1)
647				CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum",
648				    true)
649				CONF_HANDLE_SSIZE_T(opt_lg_prof_interval,
650				    "lg_prof_interval", -1,
651				    (sizeof(uint64_t) << 3) - 1)
652				CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump",
653				    true)
654				CONF_HANDLE_BOOL(opt_prof_final, "prof_final",
655				    true)
656				CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak",
657				    true)
658			}
659			malloc_conf_error("Invalid conf pair", k, klen, v,
660			    vlen);
661#undef CONF_MATCH
662#undef CONF_HANDLE_BOOL
663#undef CONF_HANDLE_SIZE_T
664#undef CONF_HANDLE_SSIZE_T
665#undef CONF_HANDLE_CHAR_P
666		}
667	}
668}
669
670static bool
671malloc_init_hard(void)
672{
673	arena_t *init_arenas[1];
674
675	malloc_mutex_lock(&init_lock);
676	if (malloc_initialized || IS_INITIALIZER) {
677		/*
678		 * Another thread initialized the allocator before this one
679		 * acquired init_lock, or this thread is the initializing
680		 * thread, and it is recursively allocating.
681		 */
682		malloc_mutex_unlock(&init_lock);
683		return (false);
684	}
685#ifdef JEMALLOC_THREADED_INIT
686	if (malloc_initializer != NO_INITIALIZER && IS_INITIALIZER == false) {
687		/* Busy-wait until the initializing thread completes. */
688		do {
689			malloc_mutex_unlock(&init_lock);
690			CPU_SPINWAIT;
691			malloc_mutex_lock(&init_lock);
692		} while (malloc_initialized == false);
693		malloc_mutex_unlock(&init_lock);
694		return (false);
695	}
696#endif
697	malloc_initializer = INITIALIZER;
698
699	malloc_tsd_boot();
700	if (config_prof)
701		prof_boot0();
702
703	malloc_conf_init();
704
705	if (opt_stats_print) {
706		/* Print statistics at exit. */
707		if (atexit(stats_print_atexit) != 0) {
708			malloc_write("<jemalloc>: Error in atexit()\n");
709			if (opt_abort)
710				abort();
711		}
712	}
713
714	if (base_boot()) {
715		malloc_mutex_unlock(&init_lock);
716		return (true);
717	}
718
719	if (chunk_boot()) {
720		malloc_mutex_unlock(&init_lock);
721		return (true);
722	}
723
724	if (ctl_boot()) {
725		malloc_mutex_unlock(&init_lock);
726		return (true);
727	}
728
729	if (config_prof)
730		prof_boot1();
731
732	arena_boot();
733
734	if (config_tcache && tcache_boot0()) {
735		malloc_mutex_unlock(&init_lock);
736		return (true);
737	}
738
739	if (huge_boot()) {
740		malloc_mutex_unlock(&init_lock);
741		return (true);
742	}
743
744	if (malloc_mutex_init(&arenas_lock)) {
745		malloc_mutex_unlock(&init_lock);
746		return (true);
747	}
748
749	/*
750	 * Create enough scaffolding to allow recursive allocation in
751	 * malloc_ncpus().
752	 */
753	narenas_total = narenas_auto = 1;
754	arenas = init_arenas;
755	memset(arenas, 0, sizeof(arena_t *) * narenas_auto);
756
757	/*
758	 * Initialize one arena here.  The rest are lazily created in
759	 * choose_arena_hard().
760	 */
761	arenas_extend(0);
762	if (arenas[0] == NULL) {
763		malloc_mutex_unlock(&init_lock);
764		return (true);
765	}
766
767	/* Initialize allocation counters before any allocations can occur. */
768	if (config_stats && thread_allocated_tsd_boot()) {
769		malloc_mutex_unlock(&init_lock);
770		return (true);
771	}
772
773	if (arenas_tsd_boot()) {
774		malloc_mutex_unlock(&init_lock);
775		return (true);
776	}
777
778	if (config_tcache && tcache_boot1()) {
779		malloc_mutex_unlock(&init_lock);
780		return (true);
781	}
782
783	if (config_fill && quarantine_boot()) {
784		malloc_mutex_unlock(&init_lock);
785		return (true);
786	}
787
788	if (config_prof && prof_boot2()) {
789		malloc_mutex_unlock(&init_lock);
790		return (true);
791	}
792
793	malloc_mutex_unlock(&init_lock);
794	/**********************************************************************/
795	/* Recursive allocation may follow. */
796
797	ncpus = malloc_ncpus();
798
799#if (!defined(JEMALLOC_MUTEX_INIT_CB) && !defined(JEMALLOC_ZONE) \
800    && !defined(_WIN32) && !defined(__native_client__))
801	/* LinuxThreads's pthread_atfork() allocates. */
802	if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent,
803	    jemalloc_postfork_child) != 0) {
804		malloc_write("<jemalloc>: Error in pthread_atfork()\n");
805		if (opt_abort)
806			abort();
807	}
808#endif
809
810	/* Done recursively allocating. */
811	/**********************************************************************/
812	malloc_mutex_lock(&init_lock);
813
814	if (mutex_boot()) {
815		malloc_mutex_unlock(&init_lock);
816		return (true);
817	}
818
819	if (opt_narenas == 0) {
820		/*
821		 * For SMP systems, create more than one arena per CPU by
822		 * default.
823		 */
824		if (ncpus > 1)
825			opt_narenas = ncpus << 2;
826		else
827			opt_narenas = 1;
828	}
829#if defined(ANDROID_MAX_ARENAS)
830	/* Never create more than MAX_ARENAS arenas regardless of num_cpus.
831	 * Extra arenas use more PSS and are not very useful unless
832	 * lots of threads are allocing/freeing at the same time.
833	 */
834	if (opt_narenas > ANDROID_MAX_ARENAS)
835		opt_narenas = ANDROID_MAX_ARENAS;
836#endif
837	narenas_auto = opt_narenas;
838	/*
839	 * Make sure that the arenas array can be allocated.  In practice, this
840	 * limit is enough to allow the allocator to function, but the ctl
841	 * machinery will fail to allocate memory at far lower limits.
842	 */
843	if (narenas_auto > chunksize / sizeof(arena_t *)) {
844		narenas_auto = chunksize / sizeof(arena_t *);
845		malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n",
846		    narenas_auto);
847	}
848	narenas_total = narenas_auto;
849
850	/* Allocate and initialize arenas. */
851	arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas_total);
852	if (arenas == NULL) {
853		malloc_mutex_unlock(&init_lock);
854		return (true);
855	}
856	/*
857	 * Zero the array.  In practice, this should always be pre-zeroed,
858	 * since it was just mmap()ed, but let's be sure.
859	 */
860	memset(arenas, 0, sizeof(arena_t *) * narenas_total);
861	/* Copy the pointer to the one arena that was already initialized. */
862	arenas[0] = init_arenas[0];
863
864	malloc_initialized = true;
865	malloc_mutex_unlock(&init_lock);
866
867	return (false);
868}
869
870/*
871 * End initialization functions.
872 */
873/******************************************************************************/
874/*
875 * Begin malloc(3)-compatible functions.
876 */
877
878static void *
879imalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt)
880{
881	void *p;
882
883	if (cnt == NULL)
884		return (NULL);
885	if (usize <= SMALL_MAXCLASS) {
886		p = imalloc(SMALL_MAXCLASS+1);
887		if (p == NULL)
888			return (NULL);
889		arena_prof_promoted(p, usize);
890	} else
891		p = imalloc(usize);
892
893	return (p);
894}
895
896JEMALLOC_ALWAYS_INLINE_C void *
897imalloc_prof(size_t usize)
898{
899	void *p;
900	prof_thr_cnt_t *cnt;
901
902	PROF_ALLOC_PREP(usize, cnt);
903	if ((uintptr_t)cnt != (uintptr_t)1U)
904		p = imalloc_prof_sample(usize, cnt);
905	else
906		p = imalloc(usize);
907	if (p == NULL)
908		return (NULL);
909	prof_malloc(p, usize, cnt);
910
911	return (p);
912}
913
914JEMALLOC_ALWAYS_INLINE_C void *
915imalloc_body(size_t size, size_t *usize)
916{
917
918	if (malloc_init())
919		return (NULL);
920
921	if (config_prof && opt_prof) {
922		*usize = s2u(size);
923		return (imalloc_prof(*usize));
924	}
925
926	if (config_stats || (config_valgrind && in_valgrind))
927		*usize = s2u(size);
928	return (imalloc(size));
929}
930
931void *
932je_malloc(size_t size)
933{
934	void *ret;
935	size_t usize JEMALLOC_CC_SILENCE_INIT(0);
936
937	if (size == 0)
938		size = 1;
939
940	ret = imalloc_body(size, &usize);
941	if (ret == NULL) {
942		if (config_xmalloc && opt_xmalloc) {
943			malloc_write("<jemalloc>: Error in malloc(): "
944			    "out of memory\n");
945			abort();
946		}
947		set_errno(ENOMEM);
948	}
949	if (config_stats && ret != NULL) {
950		assert(usize == isalloc(ret, config_prof));
951		thread_allocated_tsd_get()->allocated += usize;
952	}
953	UTRACE(0, size, ret);
954	JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, false);
955	return (ret);
956}
957
958static void *
959imemalign_prof_sample(size_t alignment, size_t usize, prof_thr_cnt_t *cnt)
960{
961	void *p;
962
963	if (cnt == NULL)
964		return (NULL);
965	if (usize <= SMALL_MAXCLASS) {
966		assert(sa2u(SMALL_MAXCLASS+1, alignment) != 0);
967		p = ipalloc(sa2u(SMALL_MAXCLASS+1, alignment), alignment,
968		    false);
969		if (p == NULL)
970			return (NULL);
971		arena_prof_promoted(p, usize);
972	} else
973		p = ipalloc(usize, alignment, false);
974
975	return (p);
976}
977
978JEMALLOC_ALWAYS_INLINE_C void *
979imemalign_prof(size_t alignment, size_t usize, prof_thr_cnt_t *cnt)
980{
981	void *p;
982
983	if ((uintptr_t)cnt != (uintptr_t)1U)
984		p = imemalign_prof_sample(alignment, usize, cnt);
985	else
986		p = ipalloc(usize, alignment, false);
987	if (p == NULL)
988		return (NULL);
989	prof_malloc(p, usize, cnt);
990
991	return (p);
992}
993
994JEMALLOC_ATTR(nonnull(1))
995static int
996imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment)
997{
998	int ret;
999	size_t usize;
1000	void *result;
1001
1002	assert(min_alignment != 0);
1003
1004	if (malloc_init()) {
1005		result = NULL;
1006		goto label_oom;
1007	} else {
1008		if (size == 0)
1009			size = 1;
1010
1011		/* Make sure that alignment is a large enough power of 2. */
1012		if (((alignment - 1) & alignment) != 0
1013		    || (alignment < min_alignment)) {
1014			if (config_xmalloc && opt_xmalloc) {
1015				malloc_write("<jemalloc>: Error allocating "
1016				    "aligned memory: invalid alignment\n");
1017				abort();
1018			}
1019			result = NULL;
1020			ret = EINVAL;
1021			goto label_return;
1022		}
1023
1024		usize = sa2u(size, alignment);
1025		if (usize == 0) {
1026			result = NULL;
1027			goto label_oom;
1028		}
1029
1030		if (config_prof && opt_prof) {
1031			prof_thr_cnt_t *cnt;
1032
1033			PROF_ALLOC_PREP(usize, cnt);
1034			result = imemalign_prof(alignment, usize, cnt);
1035		} else
1036			result = ipalloc(usize, alignment, false);
1037		if (result == NULL)
1038			goto label_oom;
1039	}
1040
1041	*memptr = result;
1042	ret = 0;
1043label_return:
1044	if (config_stats && result != NULL) {
1045		assert(usize == isalloc(result, config_prof));
1046		thread_allocated_tsd_get()->allocated += usize;
1047	}
1048	UTRACE(0, size, result);
1049	return (ret);
1050label_oom:
1051	assert(result == NULL);
1052	if (config_xmalloc && opt_xmalloc) {
1053		malloc_write("<jemalloc>: Error allocating aligned memory: "
1054		    "out of memory\n");
1055		abort();
1056	}
1057	ret = ENOMEM;
1058	goto label_return;
1059}
1060
1061int
1062je_posix_memalign(void **memptr, size_t alignment, size_t size)
1063{
1064	int ret = imemalign(memptr, alignment, size, sizeof(void *));
1065	JEMALLOC_VALGRIND_MALLOC(ret == 0, *memptr, isalloc(*memptr,
1066	    config_prof), false);
1067	return (ret);
1068}
1069
1070void *
1071je_aligned_alloc(size_t alignment, size_t size)
1072{
1073	void *ret;
1074	int err;
1075
1076	if ((err = imemalign(&ret, alignment, size, 1)) != 0) {
1077		ret = NULL;
1078		set_errno(err);
1079	}
1080	JEMALLOC_VALGRIND_MALLOC(err == 0, ret, isalloc(ret, config_prof),
1081	    false);
1082	return (ret);
1083}
1084
1085static void *
1086icalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt)
1087{
1088	void *p;
1089
1090	if (cnt == NULL)
1091		return (NULL);
1092	if (usize <= SMALL_MAXCLASS) {
1093		p = icalloc(SMALL_MAXCLASS+1);
1094		if (p == NULL)
1095			return (NULL);
1096		arena_prof_promoted(p, usize);
1097	} else
1098		p = icalloc(usize);
1099
1100	return (p);
1101}
1102
1103JEMALLOC_ALWAYS_INLINE_C void *
1104icalloc_prof(size_t usize, prof_thr_cnt_t *cnt)
1105{
1106	void *p;
1107
1108	if ((uintptr_t)cnt != (uintptr_t)1U)
1109		p = icalloc_prof_sample(usize, cnt);
1110	else
1111		p = icalloc(usize);
1112	if (p == NULL)
1113		return (NULL);
1114	prof_malloc(p, usize, cnt);
1115
1116	return (p);
1117}
1118
1119void *
1120je_calloc(size_t num, size_t size)
1121{
1122	void *ret;
1123	size_t num_size;
1124	size_t usize JEMALLOC_CC_SILENCE_INIT(0);
1125
1126	if (malloc_init()) {
1127		num_size = 0;
1128		ret = NULL;
1129		goto label_return;
1130	}
1131
1132	num_size = num * size;
1133	if (num_size == 0) {
1134		if (num == 0 || size == 0)
1135			num_size = 1;
1136		else {
1137			ret = NULL;
1138			goto label_return;
1139		}
1140	/*
1141	 * Try to avoid division here.  We know that it isn't possible to
1142	 * overflow during multiplication if neither operand uses any of the
1143	 * most significant half of the bits in a size_t.
1144	 */
1145	} else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2)))
1146	    && (num_size / size != num)) {
1147		/* size_t overflow. */
1148		ret = NULL;
1149		goto label_return;
1150	}
1151
1152	if (config_prof && opt_prof) {
1153		prof_thr_cnt_t *cnt;
1154
1155		usize = s2u(num_size);
1156		PROF_ALLOC_PREP(usize, cnt);
1157		ret = icalloc_prof(usize, cnt);
1158	} else {
1159		if (config_stats || (config_valgrind && in_valgrind))
1160			usize = s2u(num_size);
1161		ret = icalloc(num_size);
1162	}
1163
1164label_return:
1165	if (ret == NULL) {
1166		if (config_xmalloc && opt_xmalloc) {
1167			malloc_write("<jemalloc>: Error in calloc(): out of "
1168			    "memory\n");
1169			abort();
1170		}
1171		set_errno(ENOMEM);
1172	}
1173	if (config_stats && ret != NULL) {
1174		assert(usize == isalloc(ret, config_prof));
1175		thread_allocated_tsd_get()->allocated += usize;
1176	}
1177	UTRACE(0, num_size, ret);
1178	JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, true);
1179	return (ret);
1180}
1181
1182static void *
1183irealloc_prof_sample(void *oldptr, size_t usize, prof_thr_cnt_t *cnt)
1184{
1185	void *p;
1186
1187	if (cnt == NULL)
1188		return (NULL);
1189	if (usize <= SMALL_MAXCLASS) {
1190		p = iralloc(oldptr, SMALL_MAXCLASS+1, 0, 0, false);
1191		if (p == NULL)
1192			return (NULL);
1193		arena_prof_promoted(p, usize);
1194	} else
1195		p = iralloc(oldptr, usize, 0, 0, false);
1196
1197	return (p);
1198}
1199
1200JEMALLOC_ALWAYS_INLINE_C void *
1201irealloc_prof(void *oldptr, size_t old_usize, size_t usize, prof_thr_cnt_t *cnt)
1202{
1203	void *p;
1204	prof_ctx_t *old_ctx;
1205
1206	old_ctx = prof_ctx_get(oldptr);
1207	if ((uintptr_t)cnt != (uintptr_t)1U)
1208		p = irealloc_prof_sample(oldptr, usize, cnt);
1209	else
1210		p = iralloc(oldptr, usize, 0, 0, false);
1211	if (p == NULL)
1212		return (NULL);
1213	prof_realloc(p, usize, cnt, old_usize, old_ctx);
1214
1215	return (p);
1216}
1217
1218JEMALLOC_INLINE_C void
1219ifree(void *ptr)
1220{
1221	size_t usize;
1222	UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
1223
1224	assert(ptr != NULL);
1225	assert(malloc_initialized || IS_INITIALIZER);
1226
1227	if (config_prof && opt_prof) {
1228		usize = isalloc(ptr, config_prof);
1229		prof_free(ptr, usize);
1230	} else if (config_stats || config_valgrind)
1231		usize = isalloc(ptr, config_prof);
1232	if (config_stats)
1233		thread_allocated_tsd_get()->deallocated += usize;
1234	if (config_valgrind && in_valgrind)
1235		rzsize = p2rz(ptr);
1236	iqalloc(ptr);
1237	JEMALLOC_VALGRIND_FREE(ptr, rzsize);
1238}
1239
1240void *
1241je_realloc(void *ptr, size_t size)
1242{
1243	void *ret;
1244	size_t usize JEMALLOC_CC_SILENCE_INIT(0);
1245	size_t old_usize = 0;
1246	UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
1247
1248	if (size == 0) {
1249		if (ptr != NULL) {
1250			/* realloc(ptr, 0) is equivalent to free(ptr). */
1251			UTRACE(ptr, 0, 0);
1252			ifree(ptr);
1253			return (NULL);
1254		}
1255		size = 1;
1256	}
1257
1258	if (ptr != NULL) {
1259		assert(malloc_initialized || IS_INITIALIZER);
1260		malloc_thread_init();
1261
1262		if ((config_prof && opt_prof) || config_stats ||
1263		    (config_valgrind && in_valgrind))
1264			old_usize = isalloc(ptr, config_prof);
1265		if (config_valgrind && in_valgrind)
1266			old_rzsize = config_prof ? p2rz(ptr) : u2rz(old_usize);
1267
1268		if (config_prof && opt_prof) {
1269			prof_thr_cnt_t *cnt;
1270
1271			usize = s2u(size);
1272			PROF_ALLOC_PREP(usize, cnt);
1273			ret = irealloc_prof(ptr, old_usize, usize, cnt);
1274		} else {
1275			if (config_stats || (config_valgrind && in_valgrind))
1276				usize = s2u(size);
1277			ret = iralloc(ptr, size, 0, 0, false);
1278		}
1279	} else {
1280		/* realloc(NULL, size) is equivalent to malloc(size). */
1281		ret = imalloc_body(size, &usize);
1282	}
1283
1284	if (ret == NULL) {
1285		if (config_xmalloc && opt_xmalloc) {
1286			malloc_write("<jemalloc>: Error in realloc(): "
1287			    "out of memory\n");
1288			abort();
1289		}
1290		set_errno(ENOMEM);
1291	}
1292	if (config_stats && ret != NULL) {
1293		thread_allocated_t *ta;
1294		assert(usize == isalloc(ret, config_prof));
1295		ta = thread_allocated_tsd_get();
1296		ta->allocated += usize;
1297		ta->deallocated += old_usize;
1298	}
1299	UTRACE(ptr, size, ret);
1300	JEMALLOC_VALGRIND_REALLOC(true, ret, usize, true, ptr, old_usize,
1301	    old_rzsize, true, false);
1302	return (ret);
1303}
1304
1305void
1306je_free(void *ptr)
1307{
1308
1309	UTRACE(ptr, 0, 0);
1310	if (ptr != NULL)
1311		ifree(ptr);
1312}
1313
1314/*
1315 * End malloc(3)-compatible functions.
1316 */
1317/******************************************************************************/
1318/*
1319 * Begin non-standard override functions.
1320 */
1321
1322#ifdef JEMALLOC_OVERRIDE_MEMALIGN
1323void *
1324je_memalign(size_t alignment, size_t size)
1325{
1326	void *ret JEMALLOC_CC_SILENCE_INIT(NULL);
1327	imemalign(&ret, alignment, size, 1);
1328	JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false);
1329	return (ret);
1330}
1331#endif
1332
1333#ifdef JEMALLOC_OVERRIDE_VALLOC
1334void *
1335je_valloc(size_t size)
1336{
1337	void *ret JEMALLOC_CC_SILENCE_INIT(NULL);
1338	imemalign(&ret, PAGE, size, 1);
1339	JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false);
1340	return (ret);
1341}
1342#endif
1343
1344/*
1345 * is_malloc(je_malloc) is some macro magic to detect if jemalloc_defs.h has
1346 * #define je_malloc malloc
1347 */
1348#define	malloc_is_malloc 1
1349#define	is_malloc_(a) malloc_is_ ## a
1350#define	is_malloc(a) is_malloc_(a)
1351
1352#if ((is_malloc(je_malloc) == 1) && defined(__GLIBC__) && !defined(__UCLIBC__))
1353/*
1354 * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible
1355 * to inconsistently reference libc's malloc(3)-compatible functions
1356 * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541).
1357 *
1358 * These definitions interpose hooks in glibc.  The functions are actually
1359 * passed an extra argument for the caller return address, which will be
1360 * ignored.
1361 */
1362JEMALLOC_EXPORT void (*__free_hook)(void *ptr) = je_free;
1363JEMALLOC_EXPORT void *(*__malloc_hook)(size_t size) = je_malloc;
1364JEMALLOC_EXPORT void *(*__realloc_hook)(void *ptr, size_t size) = je_realloc;
1365JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) =
1366    je_memalign;
1367#endif
1368
1369/*
1370 * End non-standard override functions.
1371 */
1372/******************************************************************************/
1373/*
1374 * Begin non-standard functions.
1375 */
1376
1377JEMALLOC_ALWAYS_INLINE_C void *
1378imallocx(size_t usize, size_t alignment, bool zero, bool try_tcache,
1379    arena_t *arena)
1380{
1381
1382	assert(usize == ((alignment == 0) ? s2u(usize) : sa2u(usize,
1383	    alignment)));
1384
1385	if (alignment != 0)
1386		return (ipalloct(usize, alignment, zero, try_tcache, arena));
1387	else if (zero)
1388		return (icalloct(usize, try_tcache, arena));
1389	else
1390		return (imalloct(usize, try_tcache, arena));
1391}
1392
1393static void *
1394imallocx_prof_sample(size_t usize, size_t alignment, bool zero, bool try_tcache,
1395    arena_t *arena, prof_thr_cnt_t *cnt)
1396{
1397	void *p;
1398
1399	if (cnt == NULL)
1400		return (NULL);
1401	if (usize <= SMALL_MAXCLASS) {
1402		size_t usize_promoted = (alignment == 0) ?
1403		    s2u(SMALL_MAXCLASS+1) : sa2u(SMALL_MAXCLASS+1, alignment);
1404		assert(usize_promoted != 0);
1405		p = imallocx(usize_promoted, alignment, zero, try_tcache,
1406		    arena);
1407		if (p == NULL)
1408			return (NULL);
1409		arena_prof_promoted(p, usize);
1410	} else
1411		p = imallocx(usize, alignment, zero, try_tcache, arena);
1412
1413	return (p);
1414}
1415
1416JEMALLOC_ALWAYS_INLINE_C void *
1417imallocx_prof(size_t usize, size_t alignment, bool zero, bool try_tcache,
1418    arena_t *arena, prof_thr_cnt_t *cnt)
1419{
1420	void *p;
1421
1422	if ((uintptr_t)cnt != (uintptr_t)1U) {
1423		p = imallocx_prof_sample(usize, alignment, zero, try_tcache,
1424		    arena, cnt);
1425	} else
1426		p = imallocx(usize, alignment, zero, try_tcache, arena);
1427	if (p == NULL)
1428		return (NULL);
1429	prof_malloc(p, usize, cnt);
1430
1431	return (p);
1432}
1433
1434void *
1435je_mallocx(size_t size, int flags)
1436{
1437	void *p;
1438	size_t usize;
1439	size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK)
1440	    & (SIZE_T_MAX-1));
1441	bool zero = flags & MALLOCX_ZERO;
1442	unsigned arena_ind = ((unsigned)(flags >> 8)) - 1;
1443	arena_t *arena;
1444	bool try_tcache;
1445
1446	assert(size != 0);
1447
1448	if (malloc_init())
1449		goto label_oom;
1450
1451	if (arena_ind != UINT_MAX) {
1452		arena = arenas[arena_ind];
1453		try_tcache = false;
1454	} else {
1455		arena = NULL;
1456		try_tcache = true;
1457	}
1458
1459	usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
1460	assert(usize != 0);
1461
1462	if (config_prof && opt_prof) {
1463		prof_thr_cnt_t *cnt;
1464
1465		PROF_ALLOC_PREP(usize, cnt);
1466		p = imallocx_prof(usize, alignment, zero, try_tcache, arena,
1467		    cnt);
1468	} else
1469		p = imallocx(usize, alignment, zero, try_tcache, arena);
1470	if (p == NULL)
1471		goto label_oom;
1472
1473	if (config_stats) {
1474		assert(usize == isalloc(p, config_prof));
1475		thread_allocated_tsd_get()->allocated += usize;
1476	}
1477	UTRACE(0, size, p);
1478	JEMALLOC_VALGRIND_MALLOC(true, p, usize, zero);
1479	return (p);
1480label_oom:
1481	if (config_xmalloc && opt_xmalloc) {
1482		malloc_write("<jemalloc>: Error in mallocx(): out of memory\n");
1483		abort();
1484	}
1485	UTRACE(0, size, 0);
1486	return (NULL);
1487}
1488
1489static void *
1490irallocx_prof_sample(void *oldptr, size_t size, size_t alignment, size_t usize,
1491    bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena,
1492    prof_thr_cnt_t *cnt)
1493{
1494	void *p;
1495
1496	if (cnt == NULL)
1497		return (NULL);
1498	if (usize <= SMALL_MAXCLASS) {
1499		p = iralloct(oldptr, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >=
1500		    size) ? 0 : size - (SMALL_MAXCLASS+1), alignment, zero,
1501		    try_tcache_alloc, try_tcache_dalloc, arena);
1502		if (p == NULL)
1503			return (NULL);
1504		arena_prof_promoted(p, usize);
1505	} else {
1506		p = iralloct(oldptr, size, 0, alignment, zero,
1507		    try_tcache_alloc, try_tcache_dalloc, arena);
1508	}
1509
1510	return (p);
1511}
1512
1513JEMALLOC_ALWAYS_INLINE_C void *
1514irallocx_prof(void *oldptr, size_t old_usize, size_t size, size_t alignment,
1515    size_t *usize, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc,
1516    arena_t *arena, prof_thr_cnt_t *cnt)
1517{
1518	void *p;
1519	prof_ctx_t *old_ctx;
1520
1521	old_ctx = prof_ctx_get(oldptr);
1522	if ((uintptr_t)cnt != (uintptr_t)1U)
1523		p = irallocx_prof_sample(oldptr, size, alignment, *usize, zero,
1524		    try_tcache_alloc, try_tcache_dalloc, arena, cnt);
1525	else {
1526		p = iralloct(oldptr, size, 0, alignment, zero,
1527		    try_tcache_alloc, try_tcache_dalloc, arena);
1528	}
1529	if (p == NULL)
1530		return (NULL);
1531
1532	if (p == oldptr && alignment != 0) {
1533		/*
1534		 * The allocation did not move, so it is possible that the size
1535		 * class is smaller than would guarantee the requested
1536		 * alignment, and that the alignment constraint was
1537		 * serendipitously satisfied.  Additionally, old_usize may not
1538		 * be the same as the current usize because of in-place large
1539		 * reallocation.  Therefore, query the actual value of usize.
1540		 */
1541		*usize = isalloc(p, config_prof);
1542	}
1543	prof_realloc(p, *usize, cnt, old_usize, old_ctx);
1544
1545	return (p);
1546}
1547
1548void *
1549je_rallocx(void *ptr, size_t size, int flags)
1550{
1551	void *p;
1552	size_t usize, old_usize;
1553	UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
1554	size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK)
1555	    & (SIZE_T_MAX-1));
1556	bool zero = flags & MALLOCX_ZERO;
1557	unsigned arena_ind = ((unsigned)(flags >> 8)) - 1;
1558	bool try_tcache_alloc, try_tcache_dalloc;
1559	arena_t *arena;
1560
1561	assert(ptr != NULL);
1562	assert(size != 0);
1563	assert(malloc_initialized || IS_INITIALIZER);
1564	malloc_thread_init();
1565
1566	if (arena_ind != UINT_MAX) {
1567		arena_chunk_t *chunk;
1568		try_tcache_alloc = false;
1569		chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
1570		try_tcache_dalloc = (chunk == ptr || chunk->arena !=
1571		    arenas[arena_ind]);
1572		arena = arenas[arena_ind];
1573	} else {
1574		try_tcache_alloc = true;
1575		try_tcache_dalloc = true;
1576		arena = NULL;
1577	}
1578
1579	if ((config_prof && opt_prof) || config_stats ||
1580	    (config_valgrind && in_valgrind))
1581		old_usize = isalloc(ptr, config_prof);
1582	if (config_valgrind && in_valgrind)
1583		old_rzsize = u2rz(old_usize);
1584
1585	if (config_prof && opt_prof) {
1586		prof_thr_cnt_t *cnt;
1587
1588		usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
1589		assert(usize != 0);
1590		PROF_ALLOC_PREP(usize, cnt);
1591		p = irallocx_prof(ptr, old_usize, size, alignment, &usize, zero,
1592		    try_tcache_alloc, try_tcache_dalloc, arena, cnt);
1593		if (p == NULL)
1594			goto label_oom;
1595	} else {
1596		p = iralloct(ptr, size, 0, alignment, zero, try_tcache_alloc,
1597		    try_tcache_dalloc, arena);
1598		if (p == NULL)
1599			goto label_oom;
1600		if (config_stats || (config_valgrind && in_valgrind))
1601			usize = isalloc(p, config_prof);
1602	}
1603
1604	if (config_stats) {
1605		thread_allocated_t *ta;
1606		ta = thread_allocated_tsd_get();
1607		ta->allocated += usize;
1608		ta->deallocated += old_usize;
1609	}
1610	UTRACE(ptr, size, p);
1611	JEMALLOC_VALGRIND_REALLOC(true, p, usize, false, ptr, old_usize,
1612	    old_rzsize, false, zero);
1613	return (p);
1614label_oom:
1615	if (config_xmalloc && opt_xmalloc) {
1616		malloc_write("<jemalloc>: Error in rallocx(): out of memory\n");
1617		abort();
1618	}
1619	UTRACE(ptr, size, 0);
1620	return (NULL);
1621}
1622
1623JEMALLOC_ALWAYS_INLINE_C size_t
1624ixallocx_helper(void *ptr, size_t old_usize, size_t size, size_t extra,
1625    size_t alignment, bool zero, arena_t *arena)
1626{
1627	size_t usize;
1628
1629	if (ixalloc(ptr, size, extra, alignment, zero))
1630		return (old_usize);
1631	usize = isalloc(ptr, config_prof);
1632
1633	return (usize);
1634}
1635
1636static size_t
1637ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra,
1638    size_t alignment, size_t max_usize, bool zero, arena_t *arena,
1639    prof_thr_cnt_t *cnt)
1640{
1641	size_t usize;
1642
1643	if (cnt == NULL)
1644		return (old_usize);
1645	/* Use minimum usize to determine whether promotion may happen. */
1646	if (((alignment == 0) ? s2u(size) : sa2u(size, alignment)) <=
1647	    SMALL_MAXCLASS) {
1648		if (ixalloc(ptr, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >=
1649		    size+extra) ? 0 : size+extra - (SMALL_MAXCLASS+1),
1650		    alignment, zero))
1651			return (old_usize);
1652		usize = isalloc(ptr, config_prof);
1653		if (max_usize < PAGE)
1654			arena_prof_promoted(ptr, usize);
1655	} else {
1656		usize = ixallocx_helper(ptr, old_usize, size, extra, alignment,
1657		    zero, arena);
1658	}
1659
1660	return (usize);
1661}
1662
1663JEMALLOC_ALWAYS_INLINE_C size_t
1664ixallocx_prof(void *ptr, size_t old_usize, size_t size, size_t extra,
1665    size_t alignment, size_t max_usize, bool zero, arena_t *arena,
1666    prof_thr_cnt_t *cnt)
1667{
1668	size_t usize;
1669	prof_ctx_t *old_ctx;
1670
1671	old_ctx = prof_ctx_get(ptr);
1672	if ((uintptr_t)cnt != (uintptr_t)1U) {
1673		usize = ixallocx_prof_sample(ptr, old_usize, size, extra,
1674		    alignment, zero, max_usize, arena, cnt);
1675	} else {
1676		usize = ixallocx_helper(ptr, old_usize, size, extra, alignment,
1677		    zero, arena);
1678	}
1679	if (usize == old_usize)
1680		return (usize);
1681	prof_realloc(ptr, usize, cnt, old_usize, old_ctx);
1682
1683	return (usize);
1684}
1685
1686size_t
1687je_xallocx(void *ptr, size_t size, size_t extra, int flags)
1688{
1689	size_t usize, old_usize;
1690	UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
1691	size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK)
1692	    & (SIZE_T_MAX-1));
1693	bool zero = flags & MALLOCX_ZERO;
1694	unsigned arena_ind = ((unsigned)(flags >> 8)) - 1;
1695	arena_t *arena;
1696
1697	assert(ptr != NULL);
1698	assert(size != 0);
1699	assert(SIZE_T_MAX - size >= extra);
1700	assert(malloc_initialized || IS_INITIALIZER);
1701	malloc_thread_init();
1702
1703	if (arena_ind != UINT_MAX)
1704		arena = arenas[arena_ind];
1705	else
1706		arena = NULL;
1707
1708	old_usize = isalloc(ptr, config_prof);
1709	if (config_valgrind && in_valgrind)
1710		old_rzsize = u2rz(old_usize);
1711
1712	if (config_prof && opt_prof) {
1713		prof_thr_cnt_t *cnt;
1714		/*
1715		 * usize isn't knowable before ixalloc() returns when extra is
1716		 * non-zero.  Therefore, compute its maximum possible value and
1717		 * use that in PROF_ALLOC_PREP() to decide whether to capture a
1718		 * backtrace.  prof_realloc() will use the actual usize to
1719		 * decide whether to sample.
1720		 */
1721		size_t max_usize = (alignment == 0) ? s2u(size+extra) :
1722		    sa2u(size+extra, alignment);
1723		PROF_ALLOC_PREP(max_usize, cnt);
1724		usize = ixallocx_prof(ptr, old_usize, size, extra, alignment,
1725		    max_usize, zero, arena, cnt);
1726	} else {
1727		usize = ixallocx_helper(ptr, old_usize, size, extra, alignment,
1728		    zero, arena);
1729	}
1730	if (usize == old_usize)
1731		goto label_not_resized;
1732
1733	if (config_stats) {
1734		thread_allocated_t *ta;
1735		ta = thread_allocated_tsd_get();
1736		ta->allocated += usize;
1737		ta->deallocated += old_usize;
1738	}
1739	JEMALLOC_VALGRIND_REALLOC(false, ptr, usize, false, ptr, old_usize,
1740	    old_rzsize, false, zero);
1741label_not_resized:
1742	UTRACE(ptr, size, ptr);
1743	return (usize);
1744}
1745
1746size_t
1747je_sallocx(const void *ptr, int flags)
1748{
1749	size_t usize;
1750
1751	assert(malloc_initialized || IS_INITIALIZER);
1752	malloc_thread_init();
1753
1754	if (config_ivsalloc)
1755		usize = ivsalloc(ptr, config_prof);
1756	else {
1757		assert(ptr != NULL);
1758		usize = isalloc(ptr, config_prof);
1759	}
1760
1761	return (usize);
1762}
1763
1764void
1765je_dallocx(void *ptr, int flags)
1766{
1767	size_t usize;
1768	UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
1769	unsigned arena_ind = ((unsigned)(flags >> 8)) - 1;
1770	bool try_tcache;
1771
1772	assert(ptr != NULL);
1773	assert(malloc_initialized || IS_INITIALIZER);
1774
1775	if (arena_ind != UINT_MAX) {
1776		arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
1777		try_tcache = (chunk == ptr || chunk->arena !=
1778		    arenas[arena_ind]);
1779	} else
1780		try_tcache = true;
1781
1782	UTRACE(ptr, 0, 0);
1783	if (config_stats || config_valgrind)
1784		usize = isalloc(ptr, config_prof);
1785	if (config_prof && opt_prof) {
1786		if (config_stats == false && config_valgrind == false)
1787			usize = isalloc(ptr, config_prof);
1788		prof_free(ptr, usize);
1789	}
1790	if (config_stats)
1791		thread_allocated_tsd_get()->deallocated += usize;
1792	if (config_valgrind && in_valgrind)
1793		rzsize = p2rz(ptr);
1794	iqalloct(ptr, try_tcache);
1795	JEMALLOC_VALGRIND_FREE(ptr, rzsize);
1796}
1797
1798size_t
1799je_nallocx(size_t size, int flags)
1800{
1801	size_t usize;
1802	size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK)
1803	    & (SIZE_T_MAX-1));
1804
1805	assert(size != 0);
1806
1807	if (malloc_init())
1808		return (0);
1809
1810	usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
1811	assert(usize != 0);
1812	return (usize);
1813}
1814
1815int
1816je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp,
1817    size_t newlen)
1818{
1819
1820	if (malloc_init())
1821		return (EAGAIN);
1822
1823	return (ctl_byname(name, oldp, oldlenp, newp, newlen));
1824}
1825
1826int
1827je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp)
1828{
1829
1830	if (malloc_init())
1831		return (EAGAIN);
1832
1833	return (ctl_nametomib(name, mibp, miblenp));
1834}
1835
1836int
1837je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
1838  void *newp, size_t newlen)
1839{
1840
1841	if (malloc_init())
1842		return (EAGAIN);
1843
1844	return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen));
1845}
1846
1847void
1848je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
1849    const char *opts)
1850{
1851
1852	stats_print(write_cb, cbopaque, opts);
1853}
1854
1855size_t
1856je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr)
1857{
1858	size_t ret;
1859
1860	assert(malloc_initialized || IS_INITIALIZER);
1861	malloc_thread_init();
1862
1863	if (config_ivsalloc)
1864		ret = ivsalloc(ptr, config_prof);
1865	else
1866		ret = (ptr != NULL) ? isalloc(ptr, config_prof) : 0;
1867
1868	return (ret);
1869}
1870
1871/*
1872 * End non-standard functions.
1873 */
1874/******************************************************************************/
1875/*
1876 * The following functions are used by threading libraries for protection of
1877 * malloc during fork().
1878 */
1879
1880/*
1881 * If an application creates a thread before doing any allocation in the main
1882 * thread, then calls fork(2) in the main thread followed by memory allocation
1883 * in the child process, a race can occur that results in deadlock within the
1884 * child: the main thread may have forked while the created thread had
1885 * partially initialized the allocator.  Ordinarily jemalloc prevents
1886 * fork/malloc races via the following functions it registers during
1887 * initialization using pthread_atfork(), but of course that does no good if
1888 * the allocator isn't fully initialized at fork time.  The following library
1889 * constructor is a partial solution to this problem.  It may still possible to
1890 * trigger the deadlock described above, but doing so would involve forking via
1891 * a library constructor that runs before jemalloc's runs.
1892 */
1893JEMALLOC_ATTR(constructor)
1894static void
1895jemalloc_constructor(void)
1896{
1897
1898	malloc_init();
1899}
1900
1901#ifndef JEMALLOC_MUTEX_INIT_CB
1902void
1903jemalloc_prefork(void)
1904#else
1905JEMALLOC_EXPORT void
1906_malloc_prefork(void)
1907#endif
1908{
1909	unsigned i;
1910
1911#ifdef JEMALLOC_MUTEX_INIT_CB
1912	if (malloc_initialized == false)
1913		return;
1914#endif
1915	assert(malloc_initialized);
1916
1917	/* Acquire all mutexes in a safe order. */
1918	ctl_prefork();
1919	prof_prefork();
1920	malloc_mutex_prefork(&arenas_lock);
1921	for (i = 0; i < narenas_total; i++) {
1922		if (arenas[i] != NULL)
1923			arena_prefork(arenas[i]);
1924	}
1925	chunk_prefork();
1926	base_prefork();
1927	huge_prefork();
1928}
1929
1930#ifndef JEMALLOC_MUTEX_INIT_CB
1931void
1932jemalloc_postfork_parent(void)
1933#else
1934JEMALLOC_EXPORT void
1935_malloc_postfork(void)
1936#endif
1937{
1938	unsigned i;
1939
1940#ifdef JEMALLOC_MUTEX_INIT_CB
1941	if (malloc_initialized == false)
1942		return;
1943#endif
1944	assert(malloc_initialized);
1945
1946	/* Release all mutexes, now that fork() has completed. */
1947	huge_postfork_parent();
1948	base_postfork_parent();
1949	chunk_postfork_parent();
1950	for (i = 0; i < narenas_total; i++) {
1951		if (arenas[i] != NULL)
1952			arena_postfork_parent(arenas[i]);
1953	}
1954	malloc_mutex_postfork_parent(&arenas_lock);
1955	prof_postfork_parent();
1956	ctl_postfork_parent();
1957}
1958
1959void
1960jemalloc_postfork_child(void)
1961{
1962	unsigned i;
1963
1964	assert(malloc_initialized);
1965
1966	/* Release all mutexes, now that fork() has completed. */
1967	huge_postfork_child();
1968	base_postfork_child();
1969	chunk_postfork_child();
1970	for (i = 0; i < narenas_total; i++) {
1971		if (arenas[i] != NULL)
1972			arena_postfork_child(arenas[i]);
1973	}
1974	malloc_mutex_postfork_child(&arenas_lock);
1975	prof_postfork_child();
1976	ctl_postfork_child();
1977}
1978
1979/******************************************************************************/
1980/*
1981 * The following functions are used for TLS allocation/deallocation in static
1982 * binaries on FreeBSD.  The primary difference between these and i[mcd]alloc()
1983 * is that these avoid accessing TLS variables.
1984 */
1985
1986static void *
1987a0alloc(size_t size, bool zero)
1988{
1989
1990	if (malloc_init())
1991		return (NULL);
1992
1993	if (size == 0)
1994		size = 1;
1995
1996	if (size <= arena_maxclass)
1997		return (arena_malloc(arenas[0], size, zero, false));
1998	else
1999		return (huge_malloc(NULL, size, zero));
2000}
2001
2002void *
2003a0malloc(size_t size)
2004{
2005
2006	return (a0alloc(size, false));
2007}
2008
2009void *
2010a0calloc(size_t num, size_t size)
2011{
2012
2013	return (a0alloc(num * size, true));
2014}
2015
2016void
2017a0free(void *ptr)
2018{
2019	arena_chunk_t *chunk;
2020
2021	if (ptr == NULL)
2022		return;
2023
2024	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
2025	if (chunk != ptr)
2026		arena_dalloc(chunk, ptr, false);
2027	else
2028		huge_dalloc(ptr);
2029}
2030
2031/******************************************************************************/
2032