huge.c revision 1036ddbf11b7e9ec566b92b3dd50e105fc5f6932 
1#define	JEMALLOC_HUGE_C_
2#include "jemalloc/internal/jemalloc_internal.h"
3
4/******************************************************************************/
5/* Data. */
6
7/* Protects chunk-related data structures. */
8static malloc_mutex_t	huge_mtx;
9
10/******************************************************************************/
11
12/* Tree of chunks that are stand-alone huge allocations. */
13static extent_tree_t	huge;
14
15void *
16huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, bool try_tcache)
17{
18	size_t usize;
19
20	usize = s2u(size);
21	if (usize == 0) {
22		/* size_t overflow. */
23		return (NULL);
24	}
25
26	return (huge_palloc(tsd, arena, usize, chunksize, zero, try_tcache));
27}
28
29void *
30huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
31    bool zero, bool try_tcache)
32{
33	void *ret;
34	extent_node_t *node;
35	bool is_zeroed;
36
37	/* Allocate one or more contiguous chunks for this request. */
38
39	/* Allocate an extent node with which to track the chunk. */
40	node = ipalloct(tsd, CACHELINE_CEILING(sizeof(extent_node_t)),
41	    CACHELINE, false, try_tcache, NULL);
42	if (node == NULL)
43		return (NULL);
44
45	/*
46	 * Copy zero into is_zeroed and pass the copy to chunk_alloc(), so that
47	 * it is possible to make correct junk/zero fill decisions below.
48	 */
49	is_zeroed = zero;
50	arena = arena_choose(tsd, arena);
51	if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(arena,
52	    usize, alignment, &is_zeroed)) == NULL) {
53		idalloct(tsd, node, try_tcache);
54		return (NULL);
55	}
56
57	/* Insert node into huge. */
58	node->addr = ret;
59	node->size = usize;
60	node->zeroed = is_zeroed;
61	node->arena = arena;
62
63	malloc_mutex_lock(&huge_mtx);
64	extent_tree_ad_insert(&huge, node);
65	malloc_mutex_unlock(&huge_mtx);
66
67	if (zero || (config_fill && unlikely(opt_zero))) {
68		if (!is_zeroed)
69			memset(ret, 0, usize);
70	} else if (config_fill && unlikely(opt_junk))
71		memset(ret, 0xa5, usize);
72
73	return (ret);
74}
75
76#ifdef JEMALLOC_JET
77#undef huge_dalloc_junk
78#define	huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk_impl)
79#endif
80static void
81huge_dalloc_junk(void *ptr, size_t usize)
82{
83
84	if (config_fill && have_dss && unlikely(opt_junk)) {
85		/*
86		 * Only bother junk filling if the chunk isn't about to be
87		 * unmapped.
88		 */
89		if (!config_munmap || (have_dss && chunk_in_dss(ptr)))
90			memset(ptr, 0x5a, usize);
91	}
92}
93#ifdef JEMALLOC_JET
94#undef huge_dalloc_junk
95#define	huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk)
96huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl);
97#endif
98
99static void
100huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize,
101    size_t size, size_t extra, bool zero)
102{
103	size_t usize_next;
104	bool zeroed;
105	extent_node_t *node, key;
106	arena_t *arena;
107
108	/* Increase usize to incorporate extra. */
109	while (usize < s2u(size+extra) && (usize_next = s2u(usize+1)) < oldsize)
110		usize = usize_next;
111
112	if (oldsize == usize)
113		return;
114
115	/* Fill if necessary (shrinking). */
116	if (oldsize > usize) {
117		size_t sdiff = CHUNK_CEILING(usize) - usize;
118		zeroed = (sdiff != 0) ? !pages_purge((void *)((uintptr_t)ptr +
119		    usize), sdiff) : true;
120		if (config_fill && unlikely(opt_junk)) {
121			memset((void *)((uintptr_t)ptr + usize), 0x5a, oldsize -
122			    usize);
123			zeroed = false;
124		}
125	} else
126		zeroed = true;
127
128	malloc_mutex_lock(&huge_mtx);
129	key.addr = ptr;
130	node = extent_tree_ad_search(&huge, &key);
131	assert(node != NULL);
132	assert(node->addr == ptr);
133	arena = node->arena;
134	/* Update the size of the huge allocation. */
135	assert(node->size != usize);
136	node->size = usize;
137	/* Clear node->zeroed if zeroing failed above. */
138	node->zeroed = (node->zeroed && zeroed);
139	malloc_mutex_unlock(&huge_mtx);
140
141	arena_chunk_ralloc_huge_similar(arena, ptr, oldsize, usize);
142
143	/* Fill if necessary (growing). */
144	if (oldsize < usize) {
145		if (zero || (config_fill && unlikely(opt_zero))) {
146			if (!zeroed) {
147				memset((void *)((uintptr_t)ptr + oldsize), 0,
148				    usize - oldsize);
149			}
150		} else if (config_fill && unlikely(opt_junk)) {
151			memset((void *)((uintptr_t)ptr + oldsize), 0xa5, usize -
152			    oldsize);
153		}
154	}
155}
156
157static void
158huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize)
159{
160	size_t sdiff;
161	bool zeroed;
162	extent_node_t *node, key;
163	arena_t *arena;
164
165	sdiff = CHUNK_CEILING(usize) - usize;
166	zeroed = (sdiff != 0) ? !pages_purge((void *)((uintptr_t)ptr + usize),
167	    sdiff) : true;
168	if (config_fill && unlikely(opt_junk)) {
169		huge_dalloc_junk((void *)((uintptr_t)ptr + usize), oldsize -
170		    usize);
171		zeroed = false;
172	}
173
174	malloc_mutex_lock(&huge_mtx);
175	key.addr = ptr;
176	node = extent_tree_ad_search(&huge, &key);
177	assert(node != NULL);
178	assert(node->addr == ptr);
179	arena = node->arena;
180	/* Update the size of the huge allocation. */
181	node->size = usize;
182	/* Clear node->zeroed if zeroing failed above. */
183	node->zeroed = (node->zeroed && zeroed);
184	malloc_mutex_unlock(&huge_mtx);
185
186	/* Zap the excess chunks. */
187	arena_chunk_ralloc_huge_shrink(arena, ptr, oldsize, usize);
188}
189
190static bool
191huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) {
192	size_t usize;
193	extent_node_t *node, key;
194	arena_t *arena;
195	bool is_zeroed_subchunk, is_zeroed_chunk;
196
197	usize = s2u(size);
198	if (usize == 0) {
199		/* size_t overflow. */
200		return (true);
201	}
202
203	malloc_mutex_lock(&huge_mtx);
204	key.addr = ptr;
205	node = extent_tree_ad_search(&huge, &key);
206	assert(node != NULL);
207	assert(node->addr == ptr);
208	arena = node->arena;
209	is_zeroed_subchunk = node->zeroed;
210	malloc_mutex_unlock(&huge_mtx);
211
212	/*
213	 * Copy zero into is_zeroed_chunk and pass the copy to chunk_alloc(), so
214	 * that it is possible to make correct junk/zero fill decisions below.
215	 */
216	is_zeroed_chunk = zero;
217
218	if (arena_chunk_ralloc_huge_expand(arena, ptr, oldsize, usize,
219	     &is_zeroed_chunk))
220		return (true);
221
222	malloc_mutex_lock(&huge_mtx);
223	/* Update the size of the huge allocation. */
224	node->size = usize;
225	malloc_mutex_unlock(&huge_mtx);
226
227	if (zero || (config_fill && unlikely(opt_zero))) {
228		if (!is_zeroed_subchunk) {
229			memset((void *)((uintptr_t)ptr + oldsize), 0,
230			    CHUNK_CEILING(oldsize) - oldsize);
231		}
232		if (!is_zeroed_chunk) {
233			memset((void *)((uintptr_t)ptr +
234			    CHUNK_CEILING(oldsize)), 0, usize -
235			    CHUNK_CEILING(oldsize));
236		}
237	} else if (config_fill && unlikely(opt_junk)) {
238		memset((void *)((uintptr_t)ptr + oldsize), 0xa5, usize -
239		    oldsize);
240	}
241
242	return (false);
243}
244
245bool
246huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra,
247    bool zero)
248{
249	size_t usize;
250
251	/* Both allocations must be huge to avoid a move. */
252	if (oldsize < chunksize)
253		return (true);
254
255	assert(s2u(oldsize) == oldsize);
256	usize = s2u(size);
257	if (usize == 0) {
258		/* size_t overflow. */
259		return (true);
260	}
261
262	/*
263	 * Avoid moving the allocation if the existing chunk size accommodates
264	 * the new size.
265	 */
266	if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize)
267	    && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(size+extra)) {
268		huge_ralloc_no_move_similar(ptr, oldsize, usize, size, extra,
269		    zero);
270		return (false);
271	}
272
273	/* Shrink the allocation in-place. */
274	if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize)) {
275		huge_ralloc_no_move_shrink(ptr, oldsize, usize);
276		return (false);
277	}
278
279	/* Attempt to expand the allocation in-place. */
280	if (huge_ralloc_no_move_expand(ptr, oldsize, size + extra,
281	    zero)) {
282		if (extra == 0)
283			return (true);
284
285		/* Try again, this time without extra. */
286		return (huge_ralloc_no_move_expand(ptr, oldsize, size, zero));
287	}
288	return (false);
289}
290
291void *
292huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size,
293    size_t extra, size_t alignment, bool zero, bool try_tcache_alloc,
294    bool try_tcache_dalloc)
295{
296	void *ret;
297	size_t copysize;
298
299	/* Try to avoid moving the allocation. */
300	if (!huge_ralloc_no_move(ptr, oldsize, size, extra, zero))
301		return (ptr);
302
303	/*
304	 * size and oldsize are different enough that we need to use a
305	 * different size class.  In that case, fall back to allocating new
306	 * space and copying.
307	 */
308	if (alignment > chunksize) {
309		ret = huge_palloc(tsd, arena, size + extra, alignment, zero,
310		    try_tcache_alloc);
311	} else {
312		ret = huge_malloc(tsd, arena, size + extra, zero,
313		    try_tcache_alloc);
314	}
315
316	if (ret == NULL) {
317		if (extra == 0)
318			return (NULL);
319		/* Try again, this time without extra. */
320		if (alignment > chunksize) {
321			ret = huge_palloc(tsd, arena, size, alignment, zero,
322			    try_tcache_alloc);
323		} else {
324			ret = huge_malloc(tsd, arena, size, zero,
325			    try_tcache_alloc);
326		}
327
328		if (ret == NULL)
329			return (NULL);
330	}
331
332	/*
333	 * Copy at most size bytes (not size+extra), since the caller has no
334	 * expectation that the extra bytes will be reliably preserved.
335	 */
336	copysize = (size < oldsize) ? size : oldsize;
337	memcpy(ret, ptr, copysize);
338	isqalloc(tsd, ptr, oldsize, try_tcache_dalloc);
339	return (ret);
340}
341
342void
343huge_dalloc(tsd_t *tsd, void *ptr, bool try_tcache)
344{
345	extent_node_t *node, key;
346
347	malloc_mutex_lock(&huge_mtx);
348	/* Extract from tree of huge allocations. */
349	key.addr = ptr;
350	node = extent_tree_ad_search(&huge, &key);
351	assert(node != NULL);
352	assert(node->addr == ptr);
353	extent_tree_ad_remove(&huge, node);
354	malloc_mutex_unlock(&huge_mtx);
355
356	huge_dalloc_junk(node->addr, node->size);
357	arena_chunk_dalloc_huge(node->arena, node->addr, node->size);
358	idalloct(tsd, node, try_tcache);
359}
360
361size_t
362huge_salloc(const void *ptr)
363{
364	size_t ret;
365	extent_node_t *node, key;
366
367	malloc_mutex_lock(&huge_mtx);
368
369	/* Extract from tree of huge allocations. */
370	key.addr = __DECONST(void *, ptr);
371	node = extent_tree_ad_search(&huge, &key);
372	assert(node != NULL);
373
374	ret = node->size;
375
376	malloc_mutex_unlock(&huge_mtx);
377
378	return (ret);
379}
380
381prof_tctx_t *
382huge_prof_tctx_get(const void *ptr)
383{
384	prof_tctx_t *ret;
385	extent_node_t *node, key;
386
387	malloc_mutex_lock(&huge_mtx);
388
389	/* Extract from tree of huge allocations. */
390	key.addr = __DECONST(void *, ptr);
391	node = extent_tree_ad_search(&huge, &key);
392	assert(node != NULL);
393
394	ret = node->prof_tctx;
395
396	malloc_mutex_unlock(&huge_mtx);
397
398	return (ret);
399}
400
401void
402huge_prof_tctx_set(const void *ptr, prof_tctx_t *tctx)
403{
404	extent_node_t *node, key;
405
406	malloc_mutex_lock(&huge_mtx);
407
408	/* Extract from tree of huge allocations. */
409	key.addr = __DECONST(void *, ptr);
410	node = extent_tree_ad_search(&huge, &key);
411	assert(node != NULL);
412
413	node->prof_tctx = tctx;
414
415	malloc_mutex_unlock(&huge_mtx);
416}
417
418bool
419huge_boot(void)
420{
421
422	/* Initialize chunks data. */
423	if (malloc_mutex_init(&huge_mtx))
424		return (true);
425	extent_tree_ad_new(&huge);
426
427	return (false);
428}
429
430void
431huge_prefork(void)
432{
433
434	malloc_mutex_prefork(&huge_mtx);
435}
436
437void
438huge_postfork_parent(void)
439{
440
441	malloc_mutex_postfork_parent(&huge_mtx);
442}
443
444void
445huge_postfork_child(void)
446{
447
448	malloc_mutex_postfork_child(&huge_mtx);
449}
450