huge.c revision fb1f094f163a3bf15d8958ba845e83559c0e6dfe 
1#define	JEMALLOC_HUGE_C_
2#include "jemalloc/internal/jemalloc_internal.h"
3
4/******************************************************************************/
5
6static extent_node_t *
7huge_node_get(const void *ptr)
8{
9	extent_node_t *node;
10
11	node = chunk_lookup(ptr, true);
12	assert(!extent_node_achunk_get(node));
13
14	return (node);
15}
16
17static bool
18huge_node_set(tsdn_t *tsdn, const void *ptr, extent_node_t *node)
19{
20
21	assert(extent_node_addr_get(node) == ptr);
22	assert(!extent_node_achunk_get(node));
23	return (chunk_register(tsdn, ptr, node));
24}
25
26static void
27huge_node_reset(tsdn_t *tsdn, const void *ptr, extent_node_t *node)
28{
29	bool err;
30
31	err = huge_node_set(tsdn, ptr, node);
32	assert(!err);
33}
34
35static void
36huge_node_unset(const void *ptr, const extent_node_t *node)
37{
38
39	chunk_deregister(ptr, node);
40}
41
42void *
43huge_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero)
44{
45
46	assert(usize == s2u(usize));
47
48	return (huge_palloc(tsdn, arena, usize, chunksize, zero));
49}
50
51void *
52huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
53    bool zero)
54{
55	void *ret;
56	size_t ausize;
57	arena_t *iarena;
58	extent_node_t *node;
59	bool is_zeroed;
60
61	/* Allocate one or more contiguous chunks for this request. */
62
63	assert(!tsdn_null(tsdn) || arena != NULL);
64
65	ausize = sa2u(usize, alignment);
66	if (unlikely(ausize == 0 || ausize > HUGE_MAXCLASS))
67		return (NULL);
68	assert(ausize >= chunksize);
69
70	/* Allocate an extent node with which to track the chunk. */
71	iarena = (!tsdn_null(tsdn)) ? arena_ichoose(tsdn_tsd(tsdn), NULL) : a0get();
72	node = ipallocztm(tsdn, CACHELINE_CEILING(sizeof(extent_node_t)),
73	    CACHELINE, false, NULL, true, iarena);
74	if (node == NULL)
75		return (NULL);
76
77	/*
78	 * Copy zero into is_zeroed and pass the copy to chunk_alloc(), so that
79	 * it is possible to make correct junk/zero fill decisions below.
80	 */
81	is_zeroed = zero;
82	/* ANDROID change */
83	if (likely(!tsdn_null(tsdn))) {
84#if !defined(__LP64__)
85		/* On 32 bit systems, using a per arena cache can exhaust
86		 * virtual address space. Force all huge allocations to
87		 * always take place in the first arena.
88		 */
89		extern arena_t *a0get(void);
90		arena = a0get();
91#else
92		arena = arena_choose(tsdn_tsd(tsdn), arena);
93#endif
94	}
95	/* End ANDROID change */
96	if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(tsdn,
97	    arena, usize, alignment, &is_zeroed)) == NULL) {
98		idalloctm(tsdn, node, NULL, true, true);
99		return (NULL);
100	}
101
102	extent_node_init(node, arena, ret, usize, is_zeroed, true);
103
104	if (huge_node_set(tsdn, ret, node)) {
105		arena_chunk_dalloc_huge(tsdn, arena, ret, usize);
106		idalloctm(tsdn, node, NULL, true, true);
107		return (NULL);
108	}
109
110	/* Insert node into huge. */
111	malloc_mutex_lock(tsdn, &arena->huge_mtx);
112	ql_elm_new(node, ql_link);
113	ql_tail_insert(&arena->huge, node, ql_link);
114	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
115
116	if (zero || (config_fill && unlikely(opt_zero))) {
117		if (!is_zeroed)
118			memset(ret, 0, usize);
119	} else if (config_fill && unlikely(opt_junk_alloc))
120		memset(ret, JEMALLOC_ALLOC_JUNK, usize);
121
122	arena_decay_tick(tsdn, arena);
123	return (ret);
124}
125
126#ifdef JEMALLOC_JET
127#undef huge_dalloc_junk
128#define	huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk_impl)
129#endif
130static void
131huge_dalloc_junk(void *ptr, size_t usize)
132{
133
134	if (config_fill && have_dss && unlikely(opt_junk_free)) {
135		/*
136		 * Only bother junk filling if the chunk isn't about to be
137		 * unmapped.
138		 */
139		if (!config_munmap || (have_dss && chunk_in_dss(ptr)))
140			memset(ptr, JEMALLOC_FREE_JUNK, usize);
141	}
142}
143#ifdef JEMALLOC_JET
144#undef huge_dalloc_junk
145#define	huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk)
146huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl);
147#endif
148
149static void
150huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize,
151    size_t usize_min, size_t usize_max, bool zero)
152{
153	size_t usize, usize_next;
154	extent_node_t *node;
155	arena_t *arena;
156	chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
157	bool pre_zeroed, post_zeroed;
158
159	/* Increase usize to incorporate extra. */
160	for (usize = usize_min; usize < usize_max && (usize_next = s2u(usize+1))
161	    <= oldsize; usize = usize_next)
162		; /* Do nothing. */
163
164	if (oldsize == usize)
165		return;
166
167	node = huge_node_get(ptr);
168	arena = extent_node_arena_get(node);
169	pre_zeroed = extent_node_zeroed_get(node);
170
171	/* Fill if necessary (shrinking). */
172	if (oldsize > usize) {
173		size_t sdiff = oldsize - usize;
174		if (config_fill && unlikely(opt_junk_free)) {
175			memset((void *)((uintptr_t)ptr + usize),
176			    JEMALLOC_FREE_JUNK, sdiff);
177			post_zeroed = false;
178		} else {
179			post_zeroed = !chunk_purge_wrapper(tsdn, arena,
180			    &chunk_hooks, ptr, CHUNK_CEILING(oldsize), usize,
181			    sdiff);
182		}
183	} else
184		post_zeroed = pre_zeroed;
185
186	malloc_mutex_lock(tsdn, &arena->huge_mtx);
187	/* Update the size of the huge allocation. */
188	huge_node_unset(ptr, node);
189	assert(extent_node_size_get(node) != usize);
190	extent_node_size_set(node, usize);
191	huge_node_reset(tsdn, ptr, node);
192	/* Update zeroed. */
193	extent_node_zeroed_set(node, post_zeroed);
194	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
195
196	arena_chunk_ralloc_huge_similar(tsdn, arena, ptr, oldsize, usize);
197
198	/* Fill if necessary (growing). */
199	if (oldsize < usize) {
200		if (zero || (config_fill && unlikely(opt_zero))) {
201			if (!pre_zeroed) {
202				memset((void *)((uintptr_t)ptr + oldsize), 0,
203				    usize - oldsize);
204			}
205		} else if (config_fill && unlikely(opt_junk_alloc)) {
206			memset((void *)((uintptr_t)ptr + oldsize),
207			    JEMALLOC_ALLOC_JUNK, usize - oldsize);
208		}
209	}
210}
211
212static bool
213huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize,
214    size_t usize)
215{
216	extent_node_t *node;
217	arena_t *arena;
218	chunk_hooks_t chunk_hooks;
219	size_t cdiff;
220	bool pre_zeroed, post_zeroed;
221
222	node = huge_node_get(ptr);
223	arena = extent_node_arena_get(node);
224	pre_zeroed = extent_node_zeroed_get(node);
225	chunk_hooks = chunk_hooks_get(tsdn, arena);
226
227	assert(oldsize > usize);
228
229	/* Split excess chunks. */
230	cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize);
231	if (cdiff != 0 && chunk_hooks.split(ptr, CHUNK_CEILING(oldsize),
232	    CHUNK_CEILING(usize), cdiff, true, arena->ind))
233		return (true);
234
235	if (oldsize > usize) {
236		size_t sdiff = oldsize - usize;
237		if (config_fill && unlikely(opt_junk_free)) {
238			huge_dalloc_junk((void *)((uintptr_t)ptr + usize),
239			    sdiff);
240			post_zeroed = false;
241		} else {
242			post_zeroed = !chunk_purge_wrapper(tsdn, arena,
243			    &chunk_hooks, CHUNK_ADDR2BASE((uintptr_t)ptr +
244			    usize), CHUNK_CEILING(oldsize),
245			    CHUNK_ADDR2OFFSET((uintptr_t)ptr + usize), sdiff);
246		}
247	} else
248		post_zeroed = pre_zeroed;
249
250	malloc_mutex_lock(tsdn, &arena->huge_mtx);
251	/* Update the size of the huge allocation. */
252	huge_node_unset(ptr, node);
253	extent_node_size_set(node, usize);
254	huge_node_reset(tsdn, ptr, node);
255	/* Update zeroed. */
256	extent_node_zeroed_set(node, post_zeroed);
257	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
258
259	/* Zap the excess chunks. */
260	arena_chunk_ralloc_huge_shrink(tsdn, arena, ptr, oldsize, usize);
261
262	return (false);
263}
264
265static bool
266huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize,
267    size_t usize, bool zero) {
268	extent_node_t *node;
269	arena_t *arena;
270	bool is_zeroed_subchunk, is_zeroed_chunk;
271
272	node = huge_node_get(ptr);
273	arena = extent_node_arena_get(node);
274	malloc_mutex_lock(tsdn, &arena->huge_mtx);
275	is_zeroed_subchunk = extent_node_zeroed_get(node);
276	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
277
278	/*
279	 * Use is_zeroed_chunk to detect whether the trailing memory is zeroed,
280	 * update extent's zeroed field, and zero as necessary.
281	 */
282	is_zeroed_chunk = false;
283	if (arena_chunk_ralloc_huge_expand(tsdn, arena, ptr, oldsize, usize,
284	     &is_zeroed_chunk))
285		return (true);
286
287	malloc_mutex_lock(tsdn, &arena->huge_mtx);
288	huge_node_unset(ptr, node);
289	extent_node_size_set(node, usize);
290	extent_node_zeroed_set(node, extent_node_zeroed_get(node) &&
291	    is_zeroed_chunk);
292	huge_node_reset(tsdn, ptr, node);
293	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
294
295	if (zero || (config_fill && unlikely(opt_zero))) {
296		if (!is_zeroed_subchunk) {
297			memset((void *)((uintptr_t)ptr + oldsize), 0,
298			    CHUNK_CEILING(oldsize) - oldsize);
299		}
300		if (!is_zeroed_chunk) {
301			memset((void *)((uintptr_t)ptr +
302			    CHUNK_CEILING(oldsize)), 0, usize -
303			    CHUNK_CEILING(oldsize));
304		}
305	} else if (config_fill && unlikely(opt_junk_alloc)) {
306		memset((void *)((uintptr_t)ptr + oldsize), JEMALLOC_ALLOC_JUNK,
307		    usize - oldsize);
308	}
309
310	return (false);
311}
312
313bool
314huge_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min,
315    size_t usize_max, bool zero)
316{
317
318	assert(s2u(oldsize) == oldsize);
319	/* The following should have been caught by callers. */
320	assert(usize_min > 0 && usize_max <= HUGE_MAXCLASS);
321
322	/* Both allocations must be huge to avoid a move. */
323	if (oldsize < chunksize || usize_max < chunksize)
324		return (true);
325
326	if (CHUNK_CEILING(usize_max) > CHUNK_CEILING(oldsize)) {
327		/* Attempt to expand the allocation in-place. */
328		if (!huge_ralloc_no_move_expand(tsdn, ptr, oldsize, usize_max,
329		    zero)) {
330			arena_decay_tick(tsdn, huge_aalloc(ptr));
331			return (false);
332		}
333		/* Try again, this time with usize_min. */
334		if (usize_min < usize_max && CHUNK_CEILING(usize_min) >
335		    CHUNK_CEILING(oldsize) && huge_ralloc_no_move_expand(tsdn,
336		    ptr, oldsize, usize_min, zero)) {
337			arena_decay_tick(tsdn, huge_aalloc(ptr));
338			return (false);
339		}
340	}
341
342	/*
343	 * Avoid moving the allocation if the existing chunk size accommodates
344	 * the new size.
345	 */
346	if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize_min)
347	    && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(usize_max)) {
348		huge_ralloc_no_move_similar(tsdn, ptr, oldsize, usize_min,
349		    usize_max, zero);
350		arena_decay_tick(tsdn, huge_aalloc(ptr));
351		return (false);
352	}
353
354	/* Attempt to shrink the allocation in-place. */
355	if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(usize_max)) {
356		if (!huge_ralloc_no_move_shrink(tsdn, ptr, oldsize,
357		    usize_max)) {
358			arena_decay_tick(tsdn, huge_aalloc(ptr));
359			return (false);
360		}
361	}
362	return (true);
363}
364
365static void *
366huge_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
367    size_t alignment, bool zero)
368{
369
370	if (alignment <= chunksize)
371		return (huge_malloc(tsdn, arena, usize, zero));
372	return (huge_palloc(tsdn, arena, usize, alignment, zero));
373}
374
375void *
376huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize,
377    size_t usize, size_t alignment, bool zero, tcache_t *tcache)
378{
379	void *ret;
380	size_t copysize;
381
382	/* The following should have been caught by callers. */
383	assert(usize > 0 && usize <= HUGE_MAXCLASS);
384
385	/* Try to avoid moving the allocation. */
386	if (!huge_ralloc_no_move(tsd_tsdn(tsd), ptr, oldsize, usize, usize,
387	    zero))
388		return (ptr);
389
390	/*
391	 * usize and oldsize are different enough that we need to use a
392	 * different size class.  In that case, fall back to allocating new
393	 * space and copying.
394	 */
395	ret = huge_ralloc_move_helper(tsd_tsdn(tsd), arena, usize, alignment,
396	    zero);
397	if (ret == NULL)
398		return (NULL);
399
400	copysize = (usize < oldsize) ? usize : oldsize;
401	memcpy(ret, ptr, copysize);
402	isqalloc(tsd, ptr, oldsize, tcache, true);
403	return (ret);
404}
405
406void
407huge_dalloc(tsdn_t *tsdn, void *ptr)
408{
409	extent_node_t *node;
410	arena_t *arena;
411
412	node = huge_node_get(ptr);
413	arena = extent_node_arena_get(node);
414	huge_node_unset(ptr, node);
415	malloc_mutex_lock(tsdn, &arena->huge_mtx);
416	ql_remove(&arena->huge, node, ql_link);
417	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
418
419	huge_dalloc_junk(extent_node_addr_get(node),
420	    extent_node_size_get(node));
421	arena_chunk_dalloc_huge(tsdn, extent_node_arena_get(node),
422	    extent_node_addr_get(node), extent_node_size_get(node));
423	idalloctm(tsdn, node, NULL, true, true);
424
425	arena_decay_tick(tsdn, arena);
426}
427
428arena_t *
429huge_aalloc(const void *ptr)
430{
431
432	return (extent_node_arena_get(huge_node_get(ptr)));
433}
434
435size_t
436huge_salloc(tsdn_t *tsdn, const void *ptr)
437{
438	size_t size;
439	extent_node_t *node;
440	arena_t *arena;
441
442	node = huge_node_get(ptr);
443	arena = extent_node_arena_get(node);
444	malloc_mutex_lock(tsdn, &arena->huge_mtx);
445	size = extent_node_size_get(node);
446	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
447
448	return (size);
449}
450
451prof_tctx_t *
452huge_prof_tctx_get(tsdn_t *tsdn, const void *ptr)
453{
454	prof_tctx_t *tctx;
455	extent_node_t *node;
456	arena_t *arena;
457
458	node = huge_node_get(ptr);
459	arena = extent_node_arena_get(node);
460	malloc_mutex_lock(tsdn, &arena->huge_mtx);
461	tctx = extent_node_prof_tctx_get(node);
462	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
463
464	return (tctx);
465}
466
467void
468huge_prof_tctx_set(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx)
469{
470	extent_node_t *node;
471	arena_t *arena;
472
473	node = huge_node_get(ptr);
474	arena = extent_node_arena_get(node);
475	malloc_mutex_lock(tsdn, &arena->huge_mtx);
476	extent_node_prof_tctx_set(node, tctx);
477	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
478}
479
480void
481huge_prof_tctx_reset(tsdn_t *tsdn, const void *ptr)
482{
483
484	huge_prof_tctx_set(tsdn, ptr, (prof_tctx_t *)(uintptr_t)1U);
485}
486