1#define	JEMALLOC_HUGE_C_
2#include "jemalloc/internal/jemalloc_internal.h"
3
4/******************************************************************************/
5
6static extent_node_t *
7huge_node_get(const void *ptr)
8{
9	extent_node_t *node;
10
11	node = chunk_lookup(ptr, true);
12	assert(!extent_node_achunk_get(node));
13
14	return (node);
15}
16
17static bool
18huge_node_set(tsdn_t *tsdn, const void *ptr, extent_node_t *node)
19{
20
21	assert(extent_node_addr_get(node) == ptr);
22	assert(!extent_node_achunk_get(node));
23	return (chunk_register(tsdn, ptr, node));
24}
25
26static void
27huge_node_reset(tsdn_t *tsdn, const void *ptr, extent_node_t *node)
28{
29	bool err;
30
31	err = huge_node_set(tsdn, ptr, node);
32	assert(!err);
33}
34
35static void
36huge_node_unset(const void *ptr, const extent_node_t *node)
37{
38
39	chunk_deregister(ptr, node);
40}
41
42void *
43huge_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero)
44{
45
46	assert(usize == s2u(usize));
47
48	return (huge_palloc(tsdn, arena, usize, chunksize, zero));
49}
50
51void *
52huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
53    bool zero)
54{
55	void *ret;
56	size_t ausize;
57	arena_t *iarena;
58	extent_node_t *node;
59	size_t sn;
60	bool is_zeroed;
61
62	/* Allocate one or more contiguous chunks for this request. */
63
64	assert(!tsdn_null(tsdn) || arena != NULL);
65
66	ausize = sa2u(usize, alignment);
67	if (unlikely(ausize == 0 || ausize > HUGE_MAXCLASS))
68		return (NULL);
69	assert(ausize >= chunksize);
70
71	/* Allocate an extent node with which to track the chunk. */
72	iarena = (!tsdn_null(tsdn)) ? arena_ichoose(tsdn_tsd(tsdn), NULL) :
73	    a0get();
74	node = ipallocztm(tsdn, CACHELINE_CEILING(sizeof(extent_node_t)),
75	    CACHELINE, false, NULL, true, iarena);
76	if (node == NULL)
77		return (NULL);
78
79	/*
80	 * Copy zero into is_zeroed and pass the copy to chunk_alloc(), so that
81	 * it is possible to make correct junk/zero fill decisions below.
82	 */
83	is_zeroed = zero;
84	/* ANDROID change */
85	if (likely(!tsdn_null(tsdn))) {
86#if !defined(__LP64__)
87		/* On 32 bit systems, using a per arena cache can exhaust
88		 * virtual address space. Force all huge allocations to
89		 * always take place in the first arena.
90		 */
91		extern arena_t *a0get(void);
92		arena = a0get();
93#else
94		arena = arena_choose(tsdn_tsd(tsdn), arena);
95#endif
96	}
97	/* End ANDROID change */
98	if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(tsdn,
99	    arena, usize, alignment, &sn, &is_zeroed)) == NULL) {
100		idalloctm(tsdn, node, NULL, true, true);
101		return (NULL);
102	}
103
104	extent_node_init(node, arena, ret, usize, sn, is_zeroed, true);
105
106	if (huge_node_set(tsdn, ret, node)) {
107		arena_chunk_dalloc_huge(tsdn, arena, ret, usize, sn);
108		idalloctm(tsdn, node, NULL, true, true);
109		return (NULL);
110	}
111
112	/* Insert node into huge. */
113	malloc_mutex_lock(tsdn, &arena->huge_mtx);
114	ql_elm_new(node, ql_link);
115	ql_tail_insert(&arena->huge, node, ql_link);
116	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
117
118	if (zero || (config_fill && unlikely(opt_zero))) {
119		if (!is_zeroed)
120			memset(ret, 0, usize);
121	} else if (config_fill && unlikely(opt_junk_alloc))
122		memset(ret, JEMALLOC_ALLOC_JUNK, usize);
123
124	arena_decay_tick(tsdn, arena);
125	return (ret);
126}
127
128#ifdef JEMALLOC_JET
129#undef huge_dalloc_junk
130#define	huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk_impl)
131#endif
132static void
133huge_dalloc_junk(void *ptr, size_t usize)
134{
135
136	if (config_fill && have_dss && unlikely(opt_junk_free)) {
137		/*
138		 * Only bother junk filling if the chunk isn't about to be
139		 * unmapped.
140		 */
141		if (!config_munmap || (have_dss && chunk_in_dss(ptr)))
142			memset(ptr, JEMALLOC_FREE_JUNK, usize);
143	}
144}
145#ifdef JEMALLOC_JET
146#undef huge_dalloc_junk
147#define	huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk)
148huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl);
149#endif
150
151static void
152huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize,
153    size_t usize_min, size_t usize_max, bool zero)
154{
155	size_t usize, usize_next;
156	extent_node_t *node;
157	arena_t *arena;
158	chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
159	bool pre_zeroed, post_zeroed;
160
161	/* Increase usize to incorporate extra. */
162	for (usize = usize_min; usize < usize_max && (usize_next = s2u(usize+1))
163	    <= oldsize; usize = usize_next)
164		; /* Do nothing. */
165
166	if (oldsize == usize)
167		return;
168
169	node = huge_node_get(ptr);
170	arena = extent_node_arena_get(node);
171	pre_zeroed = extent_node_zeroed_get(node);
172
173	/* Fill if necessary (shrinking). */
174	if (oldsize > usize) {
175		size_t sdiff = oldsize - usize;
176		if (config_fill && unlikely(opt_junk_free)) {
177			memset((void *)((uintptr_t)ptr + usize),
178			    JEMALLOC_FREE_JUNK, sdiff);
179			post_zeroed = false;
180		} else {
181			post_zeroed = !chunk_purge_wrapper(tsdn, arena,
182			    &chunk_hooks, ptr, CHUNK_CEILING(oldsize), usize,
183			    sdiff);
184		}
185	} else
186		post_zeroed = pre_zeroed;
187
188	malloc_mutex_lock(tsdn, &arena->huge_mtx);
189	/* Update the size of the huge allocation. */
190	huge_node_unset(ptr, node);
191	assert(extent_node_size_get(node) != usize);
192	extent_node_size_set(node, usize);
193	huge_node_reset(tsdn, ptr, node);
194	/* Update zeroed. */
195	extent_node_zeroed_set(node, post_zeroed);
196	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
197
198	arena_chunk_ralloc_huge_similar(tsdn, arena, ptr, oldsize, usize);
199
200	/* Fill if necessary (growing). */
201	if (oldsize < usize) {
202		if (zero || (config_fill && unlikely(opt_zero))) {
203			if (!pre_zeroed) {
204				memset((void *)((uintptr_t)ptr + oldsize), 0,
205				    usize - oldsize);
206			}
207		} else if (config_fill && unlikely(opt_junk_alloc)) {
208			memset((void *)((uintptr_t)ptr + oldsize),
209			    JEMALLOC_ALLOC_JUNK, usize - oldsize);
210		}
211	}
212}
213
214static bool
215huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize,
216    size_t usize)
217{
218	extent_node_t *node;
219	arena_t *arena;
220	chunk_hooks_t chunk_hooks;
221	size_t cdiff;
222	bool pre_zeroed, post_zeroed;
223
224	node = huge_node_get(ptr);
225	arena = extent_node_arena_get(node);
226	pre_zeroed = extent_node_zeroed_get(node);
227	chunk_hooks = chunk_hooks_get(tsdn, arena);
228
229	assert(oldsize > usize);
230
231	/* Split excess chunks. */
232	cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize);
233	if (cdiff != 0 && chunk_hooks.split(ptr, CHUNK_CEILING(oldsize),
234	    CHUNK_CEILING(usize), cdiff, true, arena->ind))
235		return (true);
236
237	if (oldsize > usize) {
238		size_t sdiff = oldsize - usize;
239		if (config_fill && unlikely(opt_junk_free)) {
240			huge_dalloc_junk((void *)((uintptr_t)ptr + usize),
241			    sdiff);
242			post_zeroed = false;
243		} else {
244			post_zeroed = !chunk_purge_wrapper(tsdn, arena,
245			    &chunk_hooks, CHUNK_ADDR2BASE((uintptr_t)ptr +
246			    usize), CHUNK_CEILING(oldsize),
247			    CHUNK_ADDR2OFFSET((uintptr_t)ptr + usize), sdiff);
248		}
249	} else
250		post_zeroed = pre_zeroed;
251
252	malloc_mutex_lock(tsdn, &arena->huge_mtx);
253	/* Update the size of the huge allocation. */
254	huge_node_unset(ptr, node);
255	extent_node_size_set(node, usize);
256	huge_node_reset(tsdn, ptr, node);
257	/* Update zeroed. */
258	extent_node_zeroed_set(node, post_zeroed);
259	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
260
261	/* Zap the excess chunks. */
262	arena_chunk_ralloc_huge_shrink(tsdn, arena, ptr, oldsize, usize,
263	    extent_node_sn_get(node));
264
265	return (false);
266}
267
268static bool
269huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize,
270    size_t usize, bool zero) {
271	extent_node_t *node;
272	arena_t *arena;
273	bool is_zeroed_subchunk, is_zeroed_chunk;
274
275	node = huge_node_get(ptr);
276	arena = extent_node_arena_get(node);
277	malloc_mutex_lock(tsdn, &arena->huge_mtx);
278	is_zeroed_subchunk = extent_node_zeroed_get(node);
279	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
280
281	/*
282	 * Use is_zeroed_chunk to detect whether the trailing memory is zeroed,
283	 * update extent's zeroed field, and zero as necessary.
284	 */
285	is_zeroed_chunk = false;
286	if (arena_chunk_ralloc_huge_expand(tsdn, arena, ptr, oldsize, usize,
287	     &is_zeroed_chunk))
288		return (true);
289
290	malloc_mutex_lock(tsdn, &arena->huge_mtx);
291	huge_node_unset(ptr, node);
292	extent_node_size_set(node, usize);
293	extent_node_zeroed_set(node, extent_node_zeroed_get(node) &&
294	    is_zeroed_chunk);
295	huge_node_reset(tsdn, ptr, node);
296	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
297
298	if (zero || (config_fill && unlikely(opt_zero))) {
299		if (!is_zeroed_subchunk) {
300			memset((void *)((uintptr_t)ptr + oldsize), 0,
301			    CHUNK_CEILING(oldsize) - oldsize);
302		}
303		if (!is_zeroed_chunk) {
304			memset((void *)((uintptr_t)ptr +
305			    CHUNK_CEILING(oldsize)), 0, usize -
306			    CHUNK_CEILING(oldsize));
307		}
308	} else if (config_fill && unlikely(opt_junk_alloc)) {
309		memset((void *)((uintptr_t)ptr + oldsize), JEMALLOC_ALLOC_JUNK,
310		    usize - oldsize);
311	}
312
313	return (false);
314}
315
316bool
317huge_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min,
318    size_t usize_max, bool zero)
319{
320
321	assert(s2u(oldsize) == oldsize);
322	/* The following should have been caught by callers. */
323	assert(usize_min > 0 && usize_max <= HUGE_MAXCLASS);
324
325	/* Both allocations must be huge to avoid a move. */
326	if (oldsize < chunksize || usize_max < chunksize)
327		return (true);
328
329	if (CHUNK_CEILING(usize_max) > CHUNK_CEILING(oldsize)) {
330		/* Attempt to expand the allocation in-place. */
331		if (!huge_ralloc_no_move_expand(tsdn, ptr, oldsize, usize_max,
332		    zero)) {
333			arena_decay_tick(tsdn, huge_aalloc(ptr));
334			return (false);
335		}
336		/* Try again, this time with usize_min. */
337		if (usize_min < usize_max && CHUNK_CEILING(usize_min) >
338		    CHUNK_CEILING(oldsize) && huge_ralloc_no_move_expand(tsdn,
339		    ptr, oldsize, usize_min, zero)) {
340			arena_decay_tick(tsdn, huge_aalloc(ptr));
341			return (false);
342		}
343	}
344
345	/*
346	 * Avoid moving the allocation if the existing chunk size accommodates
347	 * the new size.
348	 */
349	if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize_min)
350	    && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(usize_max)) {
351		huge_ralloc_no_move_similar(tsdn, ptr, oldsize, usize_min,
352		    usize_max, zero);
353		arena_decay_tick(tsdn, huge_aalloc(ptr));
354		return (false);
355	}
356
357	/* Attempt to shrink the allocation in-place. */
358	if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(usize_max)) {
359		if (!huge_ralloc_no_move_shrink(tsdn, ptr, oldsize,
360		    usize_max)) {
361			arena_decay_tick(tsdn, huge_aalloc(ptr));
362			return (false);
363		}
364	}
365	return (true);
366}
367
368static void *
369huge_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
370    size_t alignment, bool zero)
371{
372
373	if (alignment <= chunksize)
374		return (huge_malloc(tsdn, arena, usize, zero));
375	return (huge_palloc(tsdn, arena, usize, alignment, zero));
376}
377
378void *
379huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize,
380    size_t usize, size_t alignment, bool zero, tcache_t *tcache)
381{
382	void *ret;
383	size_t copysize;
384
385	/* The following should have been caught by callers. */
386	assert(usize > 0 && usize <= HUGE_MAXCLASS);
387
388	/* Try to avoid moving the allocation. */
389	if (!huge_ralloc_no_move(tsd_tsdn(tsd), ptr, oldsize, usize, usize,
390	    zero))
391		return (ptr);
392
393	/*
394	 * usize and oldsize are different enough that we need to use a
395	 * different size class.  In that case, fall back to allocating new
396	 * space and copying.
397	 */
398	ret = huge_ralloc_move_helper(tsd_tsdn(tsd), arena, usize, alignment,
399	    zero);
400	if (ret == NULL)
401		return (NULL);
402
403	copysize = (usize < oldsize) ? usize : oldsize;
404	memcpy(ret, ptr, copysize);
405	isqalloc(tsd, ptr, oldsize, tcache, true);
406	return (ret);
407}
408
409void
410huge_dalloc(tsdn_t *tsdn, void *ptr)
411{
412	extent_node_t *node;
413	arena_t *arena;
414
415	node = huge_node_get(ptr);
416	arena = extent_node_arena_get(node);
417	huge_node_unset(ptr, node);
418	malloc_mutex_lock(tsdn, &arena->huge_mtx);
419	ql_remove(&arena->huge, node, ql_link);
420	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
421
422	huge_dalloc_junk(extent_node_addr_get(node),
423	    extent_node_size_get(node));
424	arena_chunk_dalloc_huge(tsdn, extent_node_arena_get(node),
425	    extent_node_addr_get(node), extent_node_size_get(node),
426	    extent_node_sn_get(node));
427	idalloctm(tsdn, node, NULL, true, true);
428
429	arena_decay_tick(tsdn, arena);
430}
431
432arena_t *
433huge_aalloc(const void *ptr)
434{
435
436	return (extent_node_arena_get(huge_node_get(ptr)));
437}
438
439size_t
440huge_salloc(tsdn_t *tsdn, const void *ptr)
441{
442	size_t size;
443	extent_node_t *node;
444	arena_t *arena;
445
446	node = huge_node_get(ptr);
447	arena = extent_node_arena_get(node);
448	malloc_mutex_lock(tsdn, &arena->huge_mtx);
449	size = extent_node_size_get(node);
450	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
451
452	return (size);
453}
454
455prof_tctx_t *
456huge_prof_tctx_get(tsdn_t *tsdn, const void *ptr)
457{
458	prof_tctx_t *tctx;
459	extent_node_t *node;
460	arena_t *arena;
461
462	node = huge_node_get(ptr);
463	arena = extent_node_arena_get(node);
464	malloc_mutex_lock(tsdn, &arena->huge_mtx);
465	tctx = extent_node_prof_tctx_get(node);
466	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
467
468	return (tctx);
469}
470
471void
472huge_prof_tctx_set(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx)
473{
474	extent_node_t *node;
475	arena_t *arena;
476
477	node = huge_node_get(ptr);
478	arena = extent_node_arena_get(node);
479	malloc_mutex_lock(tsdn, &arena->huge_mtx);
480	extent_node_prof_tctx_set(node, tctx);
481	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
482}
483
484void
485huge_prof_tctx_reset(tsdn_t *tsdn, const void *ptr)
486{
487
488	huge_prof_tctx_set(tsdn, ptr, (prof_tctx_t *)(uintptr_t)1U);
489}
490