19ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#include <stdlib.h>
29ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#include <stdarg.h>
39ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#include <stdbool.h>
49ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#include <errno.h>
59ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#include <inttypes.h>
69ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#include <math.h>
79ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#include <string.h>
89ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris
99ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#ifdef _WIN32
109ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  include <windows.h>
119ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#else
129ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  include <pthread.h>
139ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#endif
149ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris
159ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris/******************************************************************************/
169ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris/*
179ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris * Define always-enabled assertion macros, so that test assertions execute even
189ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris * if assertions are disabled in the library code.  These definitions must
199ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris * exist prior to including "jemalloc/internal/util.h".
209ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris */
219ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#define	assert(e) do {							\
229ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris	if (!(e)) {							\
239ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris		malloc_printf(						\
249ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris		    "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n",	\
259ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris		    __FILE__, __LINE__, #e);				\
269ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris		abort();						\
279ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris	}								\
289ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris} while (0)
299ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris
309ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#define	not_reached() do {						\
319ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris	malloc_printf(							\
329ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris	    "<jemalloc>: %s:%d: Unreachable code reached\n",		\
339ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris	    __FILE__, __LINE__);					\
349ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris	abort();							\
359ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris} while (0)
369ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris
379ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#define	not_implemented() do {						\
389ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris	malloc_printf("<jemalloc>: %s:%d: Not implemented\n",		\
399ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris	    __FILE__, __LINE__);					\
409ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris	abort();							\
419ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris} while (0)
429ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris
439ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#define	assert_not_implemented(e) do {					\
449ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris	if (!(e))							\
459ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris		not_implemented();					\
469ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris} while (0)
479ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris
489ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#include "test/jemalloc_test_defs.h"
499ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris
509ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#ifdef JEMALLOC_OSSPIN
519ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  include <libkern/OSAtomic.h>
529ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#endif
539ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris
549ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#if defined(HAVE_ALTIVEC) && !defined(__APPLE__)
559ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  include <altivec.h>
569ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#endif
579ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#ifdef HAVE_SSE2
589ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  include <emmintrin.h>
599ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#endif
609ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris
619ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris/******************************************************************************/
629ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris/*
639ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris * For unit tests, expose all public and private interfaces.
649ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris */
659ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#ifdef JEMALLOC_UNIT_TEST
669ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  define JEMALLOC_JET
679ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  define JEMALLOC_MANGLE
689ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  include "jemalloc/internal/jemalloc_internal.h"
699ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris
709ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris/******************************************************************************/
719ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris/*
729ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris * For integration tests, expose the public jemalloc interfaces, but only
739ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris * expose the minimum necessary internal utility code (to avoid re-implementing
749ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris * essentially identical code within the test infrastructure).
759ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris */
769ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#elif defined(JEMALLOC_INTEGRATION_TEST)
779ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  define JEMALLOC_MANGLE
789ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  include "jemalloc/jemalloc.h"
799ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  include "jemalloc/internal/jemalloc_internal_defs.h"
809ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  include "jemalloc/internal/jemalloc_internal_macros.h"
819ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris
829ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  define JEMALLOC_N(n) je_##n
839ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  include "jemalloc/internal/private_namespace.h"
849ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris
859ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  define JEMALLOC_H_TYPES
869ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  define JEMALLOC_H_STRUCTS
879ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  define JEMALLOC_H_EXTERNS
889ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  define JEMALLOC_H_INLINES
899ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  include "jemalloc/internal/util.h"
909ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  include "jemalloc/internal/qr.h"
919ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  include "jemalloc/internal/ql.h"
929ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  undef JEMALLOC_H_TYPES
939ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  undef JEMALLOC_H_STRUCTS
949ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  undef JEMALLOC_H_EXTERNS
959ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  undef JEMALLOC_H_INLINES
969ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris
979ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris/******************************************************************************/
989ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris/*
999ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris * For stress tests, expose the public jemalloc interfaces with name mangling
1009ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris * so that they can be tested as e.g. malloc() and free().  Also expose the
1019ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris * public jemalloc interfaces with jet_ prefixes, so that stress tests can use
1029ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris * a separate allocator for their internal data structures.
1039ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris */
1049ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#elif defined(JEMALLOC_STRESS_TEST)
1059ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  include "jemalloc/jemalloc.h"
1069ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris
1079ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  include "jemalloc/jemalloc_protos_jet.h"
1089ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris
1099ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  define JEMALLOC_JET
1109ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  include "jemalloc/internal/jemalloc_internal.h"
1119ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  include "jemalloc/internal/public_unnamespace.h"
1129ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  undef JEMALLOC_JET
1139ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris
1149ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  include "jemalloc/jemalloc_rename.h"
1159ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  define JEMALLOC_MANGLE
1169ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  ifdef JEMALLOC_STRESS_TESTLIB
1179ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#    include "jemalloc/jemalloc_mangle_jet.h"
1189ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  else
1199ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#    include "jemalloc/jemalloc_mangle.h"
1209ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  endif
1219ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris
1229ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris/******************************************************************************/
1239ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris/*
1249ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris * This header does dangerous things, the effects of which only test code
1259ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris * should be subject to.
1269ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris */
1279ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#else
1289ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#  error "This header cannot be included outside a testing context"
1299ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#endif
1309ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris
1319ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris/******************************************************************************/
1329ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris/*
1339ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris * Common test utilities.
1349ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris */
1359ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#include "test/math.h"
1369ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#include "test/mtx.h"
1379ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#include "test/mq.h"
1389ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#include "test/test.h"
1399ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#include "test/thd.h"
1409ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#define	MEXP 19937
1419ebe2acb722ad8e29179cfbff35547f607e0f532Christopher Ferris#include "test/SFMT.h"
142