1#define	JEMALLOC_QUARANTINE_C_
2#include "jemalloc/internal/jemalloc_internal.h"
3
4/*
5 * quarantine pointers close to NULL are used to encode state information that
6 * is used for cleaning up during thread shutdown.
7 */
8#define	QUARANTINE_STATE_REINCARNATED	((quarantine_t *)(uintptr_t)1)
9#define	QUARANTINE_STATE_PURGATORY	((quarantine_t *)(uintptr_t)2)
10#define	QUARANTINE_STATE_MAX		QUARANTINE_STATE_PURGATORY
11
12/******************************************************************************/
13/* Data. */
14
15malloc_tsd_data(, quarantine, quarantine_t *, NULL)
16
17/******************************************************************************/
18/* Function prototypes for non-inline static functions. */
19
20static quarantine_t	*quarantine_grow(quarantine_t *quarantine);
21static void	quarantine_drain_one(quarantine_t *quarantine);
22static void	quarantine_drain(quarantine_t *quarantine, size_t upper_bound);
23
24/******************************************************************************/
25
26quarantine_t *
27quarantine_init(size_t lg_maxobjs)
28{
29	quarantine_t *quarantine;
30
31	quarantine = (quarantine_t *)imalloc(offsetof(quarantine_t, objs) +
32	    ((ZU(1) << lg_maxobjs) * sizeof(quarantine_obj_t)));
33	if (quarantine == NULL)
34		return (NULL);
35	quarantine->curbytes = 0;
36	quarantine->curobjs = 0;
37	quarantine->first = 0;
38	quarantine->lg_maxobjs = lg_maxobjs;
39
40	quarantine_tsd_set(&quarantine);
41
42	return (quarantine);
43}
44
45static quarantine_t *
46quarantine_grow(quarantine_t *quarantine)
47{
48	quarantine_t *ret;
49
50	ret = quarantine_init(quarantine->lg_maxobjs + 1);
51	if (ret == NULL) {
52		quarantine_drain_one(quarantine);
53		return (quarantine);
54	}
55
56	ret->curbytes = quarantine->curbytes;
57	ret->curobjs = quarantine->curobjs;
58	if (quarantine->first + quarantine->curobjs <= (ZU(1) <<
59	    quarantine->lg_maxobjs)) {
60		/* objs ring buffer data are contiguous. */
61		memcpy(ret->objs, &quarantine->objs[quarantine->first],
62		    quarantine->curobjs * sizeof(quarantine_obj_t));
63	} else {
64		/* objs ring buffer data wrap around. */
65		size_t ncopy_a = (ZU(1) << quarantine->lg_maxobjs) -
66		    quarantine->first;
67		size_t ncopy_b = quarantine->curobjs - ncopy_a;
68
69		memcpy(ret->objs, &quarantine->objs[quarantine->first], ncopy_a
70		    * sizeof(quarantine_obj_t));
71		memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b *
72		    sizeof(quarantine_obj_t));
73	}
74	idalloc(quarantine);
75
76	return (ret);
77}
78
79static void
80quarantine_drain_one(quarantine_t *quarantine)
81{
82	quarantine_obj_t *obj = &quarantine->objs[quarantine->first];
83	assert(obj->usize == isalloc(obj->ptr, config_prof));
84	idalloc(obj->ptr);
85	quarantine->curbytes -= obj->usize;
86	quarantine->curobjs--;
87	quarantine->first = (quarantine->first + 1) & ((ZU(1) <<
88	    quarantine->lg_maxobjs) - 1);
89}
90
91static void
92quarantine_drain(quarantine_t *quarantine, size_t upper_bound)
93{
94
95	while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0)
96		quarantine_drain_one(quarantine);
97}
98
99void
100quarantine(void *ptr)
101{
102	quarantine_t *quarantine;
103	size_t usize = isalloc(ptr, config_prof);
104
105	cassert(config_fill);
106	assert(opt_quarantine);
107
108	quarantine = *quarantine_tsd_get();
109	if ((uintptr_t)quarantine <= (uintptr_t)QUARANTINE_STATE_MAX) {
110		if (quarantine == QUARANTINE_STATE_PURGATORY) {
111			/*
112			 * Make a note that quarantine() was called after
113			 * quarantine_cleanup() was called.
114			 */
115			quarantine = QUARANTINE_STATE_REINCARNATED;
116			quarantine_tsd_set(&quarantine);
117		}
118		idalloc(ptr);
119		return;
120	}
121	/*
122	 * Drain one or more objects if the quarantine size limit would be
123	 * exceeded by appending ptr.
124	 */
125	if (quarantine->curbytes + usize > opt_quarantine) {
126		size_t upper_bound = (opt_quarantine >= usize) ? opt_quarantine
127		    - usize : 0;
128		quarantine_drain(quarantine, upper_bound);
129	}
130	/* Grow the quarantine ring buffer if it's full. */
131	if (quarantine->curobjs == (ZU(1) << quarantine->lg_maxobjs))
132		quarantine = quarantine_grow(quarantine);
133	/* quarantine_grow() must free a slot if it fails to grow. */
134	assert(quarantine->curobjs < (ZU(1) << quarantine->lg_maxobjs));
135	/* Append ptr if its size doesn't exceed the quarantine size. */
136	if (quarantine->curbytes + usize <= opt_quarantine) {
137		size_t offset = (quarantine->first + quarantine->curobjs) &
138		    ((ZU(1) << quarantine->lg_maxobjs) - 1);
139		quarantine_obj_t *obj = &quarantine->objs[offset];
140		obj->ptr = ptr;
141		obj->usize = usize;
142		quarantine->curbytes += usize;
143		quarantine->curobjs++;
144		if (config_fill && opt_junk) {
145			/*
146			 * Only do redzone validation if Valgrind isn't in
147			 * operation.
148			 */
149			if ((config_valgrind == false || in_valgrind == false)
150			    && usize <= SMALL_MAXCLASS)
151				arena_quarantine_junk_small(ptr, usize);
152			else
153				memset(ptr, 0x5a, usize);
154		}
155	} else {
156		assert(quarantine->curbytes == 0);
157		idalloc(ptr);
158	}
159}
160
161void
162quarantine_cleanup(void *arg)
163{
164	quarantine_t *quarantine = *(quarantine_t **)arg;
165
166	if (quarantine == QUARANTINE_STATE_REINCARNATED) {
167		/*
168		 * Another destructor deallocated memory after this destructor
169		 * was called.  Reset quarantine to QUARANTINE_STATE_PURGATORY
170		 * in order to receive another callback.
171		 */
172		quarantine = QUARANTINE_STATE_PURGATORY;
173		quarantine_tsd_set(&quarantine);
174	} else if (quarantine == QUARANTINE_STATE_PURGATORY) {
175		/*
176		 * The previous time this destructor was called, we set the key
177		 * to QUARANTINE_STATE_PURGATORY so that other destructors
178		 * wouldn't cause re-creation of the quarantine.  This time, do
179		 * nothing, so that the destructor will not be called again.
180		 */
181	} else if (quarantine != NULL) {
182		quarantine_drain(quarantine, 0);
183		idalloc(quarantine);
184		quarantine = QUARANTINE_STATE_PURGATORY;
185		quarantine_tsd_set(&quarantine);
186	}
187}
188
189bool
190quarantine_boot(void)
191{
192
193	cassert(config_fill);
194
195	if (quarantine_tsd_boot())
196		return (true);
197
198	return (false);
199}
200