1#define	JEMALLOC_HUGE_C_
2#include "jemalloc/internal/jemalloc_internal.h"
3
4/******************************************************************************/
5
6static extent_node_t *
7huge_node_get(const void *ptr)
8{
9	extent_node_t *node;
10
11	node = chunk_lookup(ptr, true);
12	assert(!extent_node_achunk_get(node));
13
14	return (node);
15}
16
17static bool
18huge_node_set(const void *ptr, extent_node_t *node)
19{
20
21	assert(extent_node_addr_get(node) == ptr);
22	assert(!extent_node_achunk_get(node));
23	return (chunk_register(ptr, node));
24}
25
26static void
27huge_node_unset(const void *ptr, const extent_node_t *node)
28{
29
30	chunk_deregister(ptr, node);
31}
32
33void *
34huge_malloc(tsd_t *tsd, arena_t *arena, size_t usize, bool zero,
35    tcache_t *tcache)
36{
37
38	assert(usize == s2u(usize));
39
40	return (huge_palloc(tsd, arena, usize, chunksize, zero, tcache));
41}
42
43void *
44huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
45    bool zero, tcache_t *tcache)
46{
47	void *ret;
48	size_t ausize;
49	extent_node_t *node;
50	bool is_zeroed;
51
52	/* Allocate one or more contiguous chunks for this request. */
53
54	ausize = sa2u(usize, alignment);
55	if (unlikely(ausize == 0 || ausize > HUGE_MAXCLASS))
56		return (NULL);
57	assert(ausize >= chunksize);
58
59	/* Allocate an extent node with which to track the chunk. */
60	node = ipallocztm(tsd, CACHELINE_CEILING(sizeof(extent_node_t)),
61	    CACHELINE, false, tcache, true, arena);
62	if (node == NULL)
63		return (NULL);
64
65	/*
66	 * Copy zero into is_zeroed and pass the copy to chunk_alloc(), so that
67	 * it is possible to make correct junk/zero fill decisions below.
68	 */
69	is_zeroed = zero;
70	/* ANDROID change */
71#if !defined(__LP64__)
72	/* On 32 bit systems, using a per arena cache can exhaust
73	 * virtual address space. Force all huge allocations to
74	 * always take place in the first arena.
75	 */
76	extern arena_t *a0get(void);
77	arena = a0get();
78#else
79	arena = arena_choose(tsd, arena);
80#endif
81	/* End ANDROID change */
82	if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(arena,
83	    usize, alignment, &is_zeroed)) == NULL) {
84		idalloctm(tsd, node, tcache, true, true);
85		return (NULL);
86	}
87
88	extent_node_init(node, arena, ret, usize, is_zeroed, true);
89
90	if (huge_node_set(ret, node)) {
91		arena_chunk_dalloc_huge(arena, ret, usize);
92		idalloctm(tsd, node, tcache, true, true);
93		return (NULL);
94	}
95
96	/* Insert node into huge. */
97	malloc_mutex_lock(&arena->huge_mtx);
98	ql_elm_new(node, ql_link);
99	ql_tail_insert(&arena->huge, node, ql_link);
100	malloc_mutex_unlock(&arena->huge_mtx);
101
102	if (zero || (config_fill && unlikely(opt_zero))) {
103		if (!is_zeroed)
104			memset(ret, 0, usize);
105	} else if (config_fill && unlikely(opt_junk_alloc))
106		memset(ret, 0xa5, usize);
107
108	arena_decay_tick(tsd, arena);
109	return (ret);
110}
111
112#ifdef JEMALLOC_JET
113#undef huge_dalloc_junk
114#define	huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk_impl)
115#endif
116static void
117huge_dalloc_junk(void *ptr, size_t usize)
118{
119
120	if (config_fill && have_dss && unlikely(opt_junk_free)) {
121		/*
122		 * Only bother junk filling if the chunk isn't about to be
123		 * unmapped.
124		 */
125		if (!config_munmap || (have_dss && chunk_in_dss(ptr)))
126			memset(ptr, 0x5a, usize);
127	}
128}
129#ifdef JEMALLOC_JET
130#undef huge_dalloc_junk
131#define	huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk)
132huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl);
133#endif
134
135static void
136huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize_min,
137    size_t usize_max, bool zero)
138{
139	size_t usize, usize_next;
140	extent_node_t *node;
141	arena_t *arena;
142	chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
143	bool pre_zeroed, post_zeroed;
144
145	/* Increase usize to incorporate extra. */
146	for (usize = usize_min; usize < usize_max && (usize_next = s2u(usize+1))
147	    <= oldsize; usize = usize_next)
148		; /* Do nothing. */
149
150	if (oldsize == usize)
151		return;
152
153	node = huge_node_get(ptr);
154	arena = extent_node_arena_get(node);
155	pre_zeroed = extent_node_zeroed_get(node);
156
157	/* Fill if necessary (shrinking). */
158	if (oldsize > usize) {
159		size_t sdiff = oldsize - usize;
160		if (config_fill && unlikely(opt_junk_free)) {
161			memset((void *)((uintptr_t)ptr + usize), 0x5a, sdiff);
162			post_zeroed = false;
163		} else {
164			post_zeroed = !chunk_purge_wrapper(arena, &chunk_hooks,
165			    ptr, CHUNK_CEILING(oldsize), usize, sdiff);
166		}
167	} else
168		post_zeroed = pre_zeroed;
169
170	malloc_mutex_lock(&arena->huge_mtx);
171	/* Update the size of the huge allocation. */
172	assert(extent_node_size_get(node) != usize);
173	extent_node_size_set(node, usize);
174	/* Update zeroed. */
175	extent_node_zeroed_set(node, post_zeroed);
176	malloc_mutex_unlock(&arena->huge_mtx);
177
178	arena_chunk_ralloc_huge_similar(arena, ptr, oldsize, usize);
179
180	/* Fill if necessary (growing). */
181	if (oldsize < usize) {
182		if (zero || (config_fill && unlikely(opt_zero))) {
183			if (!pre_zeroed) {
184				memset((void *)((uintptr_t)ptr + oldsize), 0,
185				    usize - oldsize);
186			}
187		} else if (config_fill && unlikely(opt_junk_alloc)) {
188			memset((void *)((uintptr_t)ptr + oldsize), 0xa5, usize -
189			    oldsize);
190		}
191	}
192}
193
194static bool
195huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize)
196{
197	extent_node_t *node;
198	arena_t *arena;
199	chunk_hooks_t chunk_hooks;
200	size_t cdiff;
201	bool pre_zeroed, post_zeroed;
202
203	node = huge_node_get(ptr);
204	arena = extent_node_arena_get(node);
205	pre_zeroed = extent_node_zeroed_get(node);
206	chunk_hooks = chunk_hooks_get(arena);
207
208	assert(oldsize > usize);
209
210	/* Split excess chunks. */
211	cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize);
212	if (cdiff != 0 && chunk_hooks.split(ptr, CHUNK_CEILING(oldsize),
213	    CHUNK_CEILING(usize), cdiff, true, arena->ind))
214		return (true);
215
216	if (oldsize > usize) {
217		size_t sdiff = oldsize - usize;
218		if (config_fill && unlikely(opt_junk_free)) {
219			huge_dalloc_junk((void *)((uintptr_t)ptr + usize),
220			    sdiff);
221			post_zeroed = false;
222		} else {
223			post_zeroed = !chunk_purge_wrapper(arena, &chunk_hooks,
224			    CHUNK_ADDR2BASE((uintptr_t)ptr + usize),
225			    CHUNK_CEILING(oldsize),
226			    CHUNK_ADDR2OFFSET((uintptr_t)ptr + usize), sdiff);
227		}
228	} else
229		post_zeroed = pre_zeroed;
230
231	malloc_mutex_lock(&arena->huge_mtx);
232	/* Update the size of the huge allocation. */
233	extent_node_size_set(node, usize);
234	/* Update zeroed. */
235	extent_node_zeroed_set(node, post_zeroed);
236	malloc_mutex_unlock(&arena->huge_mtx);
237
238	/* Zap the excess chunks. */
239	arena_chunk_ralloc_huge_shrink(arena, ptr, oldsize, usize);
240
241	return (false);
242}
243
244static bool
245huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t usize, bool zero) {
246	extent_node_t *node;
247	arena_t *arena;
248	bool is_zeroed_subchunk, is_zeroed_chunk;
249
250	node = huge_node_get(ptr);
251	arena = extent_node_arena_get(node);
252	malloc_mutex_lock(&arena->huge_mtx);
253	is_zeroed_subchunk = extent_node_zeroed_get(node);
254	malloc_mutex_unlock(&arena->huge_mtx);
255
256	/*
257	 * Copy zero into is_zeroed_chunk and pass the copy to chunk_alloc(), so
258	 * that it is possible to make correct junk/zero fill decisions below.
259	 */
260	is_zeroed_chunk = zero;
261
262	if (arena_chunk_ralloc_huge_expand(arena, ptr, oldsize, usize,
263	     &is_zeroed_chunk))
264		return (true);
265
266	malloc_mutex_lock(&arena->huge_mtx);
267	/* Update the size of the huge allocation. */
268	extent_node_size_set(node, usize);
269	malloc_mutex_unlock(&arena->huge_mtx);
270
271	if (zero || (config_fill && unlikely(opt_zero))) {
272		if (!is_zeroed_subchunk) {
273			memset((void *)((uintptr_t)ptr + oldsize), 0,
274			    CHUNK_CEILING(oldsize) - oldsize);
275		}
276		if (!is_zeroed_chunk) {
277			memset((void *)((uintptr_t)ptr +
278			    CHUNK_CEILING(oldsize)), 0, usize -
279			    CHUNK_CEILING(oldsize));
280		}
281	} else if (config_fill && unlikely(opt_junk_alloc)) {
282		memset((void *)((uintptr_t)ptr + oldsize), 0xa5, usize -
283		    oldsize);
284	}
285
286	return (false);
287}
288
289bool
290huge_ralloc_no_move(tsd_t *tsd, void *ptr, size_t oldsize, size_t usize_min,
291    size_t usize_max, bool zero)
292{
293
294	assert(s2u(oldsize) == oldsize);
295	/* The following should have been caught by callers. */
296	assert(usize_min > 0 && usize_max <= HUGE_MAXCLASS);
297
298	/* Both allocations must be huge to avoid a move. */
299	if (oldsize < chunksize || usize_max < chunksize)
300		return (true);
301
302	if (CHUNK_CEILING(usize_max) > CHUNK_CEILING(oldsize)) {
303		/* Attempt to expand the allocation in-place. */
304		if (!huge_ralloc_no_move_expand(ptr, oldsize, usize_max,
305		    zero)) {
306			arena_decay_tick(tsd, huge_aalloc(ptr));
307			return (false);
308		}
309		/* Try again, this time with usize_min. */
310		if (usize_min < usize_max && CHUNK_CEILING(usize_min) >
311		    CHUNK_CEILING(oldsize) && huge_ralloc_no_move_expand(ptr,
312		    oldsize, usize_min, zero)) {
313			arena_decay_tick(tsd, huge_aalloc(ptr));
314			return (false);
315		}
316	}
317
318	/*
319	 * Avoid moving the allocation if the existing chunk size accommodates
320	 * the new size.
321	 */
322	if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize_min)
323	    && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(usize_max)) {
324		huge_ralloc_no_move_similar(ptr, oldsize, usize_min, usize_max,
325		    zero);
326		arena_decay_tick(tsd, huge_aalloc(ptr));
327		return (false);
328	}
329
330	/* Attempt to shrink the allocation in-place. */
331	if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(usize_max)) {
332		if (!huge_ralloc_no_move_shrink(ptr, oldsize, usize_max)) {
333			arena_decay_tick(tsd, huge_aalloc(ptr));
334			return (false);
335		}
336	}
337	return (true);
338}
339
340static void *
341huge_ralloc_move_helper(tsd_t *tsd, arena_t *arena, size_t usize,
342    size_t alignment, bool zero, tcache_t *tcache)
343{
344
345	if (alignment <= chunksize)
346		return (huge_malloc(tsd, arena, usize, zero, tcache));
347	return (huge_palloc(tsd, arena, usize, alignment, zero, tcache));
348}
349
350void *
351huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t usize,
352    size_t alignment, bool zero, tcache_t *tcache)
353{
354	void *ret;
355	size_t copysize;
356
357	/* The following should have been caught by callers. */
358	assert(usize > 0 && usize <= HUGE_MAXCLASS);
359
360	/* Try to avoid moving the allocation. */
361	if (!huge_ralloc_no_move(tsd, ptr, oldsize, usize, usize, zero))
362		return (ptr);
363
364	/*
365	 * usize and oldsize are different enough that we need to use a
366	 * different size class.  In that case, fall back to allocating new
367	 * space and copying.
368	 */
369	ret = huge_ralloc_move_helper(tsd, arena, usize, alignment, zero,
370	    tcache);
371	if (ret == NULL)
372		return (NULL);
373
374	copysize = (usize < oldsize) ? usize : oldsize;
375	memcpy(ret, ptr, copysize);
376	isqalloc(tsd, ptr, oldsize, tcache);
377	return (ret);
378}
379
380void
381huge_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache)
382{
383	extent_node_t *node;
384	arena_t *arena;
385
386	node = huge_node_get(ptr);
387	arena = extent_node_arena_get(node);
388	huge_node_unset(ptr, node);
389	malloc_mutex_lock(&arena->huge_mtx);
390	ql_remove(&arena->huge, node, ql_link);
391	malloc_mutex_unlock(&arena->huge_mtx);
392
393	huge_dalloc_junk(extent_node_addr_get(node),
394	    extent_node_size_get(node));
395	arena_chunk_dalloc_huge(extent_node_arena_get(node),
396	    extent_node_addr_get(node), extent_node_size_get(node));
397	idalloctm(tsd, node, tcache, true, true);
398
399	arena_decay_tick(tsd, arena);
400}
401
402arena_t *
403huge_aalloc(const void *ptr)
404{
405
406	return (extent_node_arena_get(huge_node_get(ptr)));
407}
408
409size_t
410huge_salloc(const void *ptr)
411{
412	size_t size;
413	extent_node_t *node;
414	arena_t *arena;
415
416	node = huge_node_get(ptr);
417	arena = extent_node_arena_get(node);
418	malloc_mutex_lock(&arena->huge_mtx);
419	size = extent_node_size_get(node);
420	malloc_mutex_unlock(&arena->huge_mtx);
421
422	return (size);
423}
424
425prof_tctx_t *
426huge_prof_tctx_get(const void *ptr)
427{
428	prof_tctx_t *tctx;
429	extent_node_t *node;
430	arena_t *arena;
431
432	node = huge_node_get(ptr);
433	arena = extent_node_arena_get(node);
434	malloc_mutex_lock(&arena->huge_mtx);
435	tctx = extent_node_prof_tctx_get(node);
436	malloc_mutex_unlock(&arena->huge_mtx);
437
438	return (tctx);
439}
440
441void
442huge_prof_tctx_set(const void *ptr, prof_tctx_t *tctx)
443{
444	extent_node_t *node;
445	arena_t *arena;
446
447	node = huge_node_get(ptr);
448	arena = extent_node_arena_get(node);
449	malloc_mutex_lock(&arena->huge_mtx);
450	extent_node_prof_tctx_set(node, tctx);
451	malloc_mutex_unlock(&arena->huge_mtx);
452}
453
454void
455huge_prof_tctx_reset(const void *ptr)
456{
457
458	huge_prof_tctx_set(ptr, (prof_tctx_t *)(uintptr_t)1U);
459}
460