1/******************************************************************************/
2#ifdef JEMALLOC_H_TYPES
3
4typedef struct tcache_bin_info_s tcache_bin_info_t;
5typedef struct tcache_bin_s tcache_bin_t;
6typedef struct tcache_s tcache_t;
7typedef struct tcaches_s tcaches_t;
8
9/*
10 * tcache pointers close to NULL are used to encode state information that is
11 * used for two purposes: preventing thread caching on a per thread basis and
12 * cleaning up during thread shutdown.
13 */
14#define	TCACHE_STATE_DISABLED		((tcache_t *)(uintptr_t)1)
15#define	TCACHE_STATE_REINCARNATED	((tcache_t *)(uintptr_t)2)
16#define	TCACHE_STATE_PURGATORY		((tcache_t *)(uintptr_t)3)
17#define	TCACHE_STATE_MAX		TCACHE_STATE_PURGATORY
18
19/*
20 * Absolute minimum number of cache slots for each small bin.
21 */
22#define	TCACHE_NSLOTS_SMALL_MIN		20
23
24/*
25 * Absolute maximum number of cache slots for each small bin in the thread
26 * cache.  This is an additional constraint beyond that imposed as: twice the
27 * number of regions per run for this size class.
28 *
29 * This constant must be an even number.
30 */
31#if defined(ANDROID_TCACHE_NSLOTS_SMALL_MAX)
32#define	TCACHE_NSLOTS_SMALL_MAX		ANDROID_TCACHE_NSLOTS_SMALL_MAX
33#else
34#define	TCACHE_NSLOTS_SMALL_MAX		200
35#endif
36
37/* Number of cache slots for large size classes. */
38#if defined(ANDROID_TCACHE_NSLOTS_LARGE)
39#define	TCACHE_NSLOTS_LARGE		ANDROID_TCACHE_NSLOTS_LARGE
40#else
41#define	TCACHE_NSLOTS_LARGE		20
42#endif
43
44/* (1U << opt_lg_tcache_max) is used to compute tcache_maxclass. */
45#if defined(ANDROID_LG_TCACHE_MAXCLASS_DEFAULT)
46#define	LG_TCACHE_MAXCLASS_DEFAULT	ANDROID_LG_TCACHE_MAXCLASS_DEFAULT
47#else
48#define	LG_TCACHE_MAXCLASS_DEFAULT	15
49#endif
50
51/*
52 * TCACHE_GC_SWEEP is the approximate number of allocation events between
53 * full GC sweeps.  Integer rounding may cause the actual number to be
54 * slightly higher, since GC is performed incrementally.
55 */
56#define	TCACHE_GC_SWEEP			8192
57
58/* Number of tcache allocation/deallocation events between incremental GCs. */
59#define	TCACHE_GC_INCR							\
60    ((TCACHE_GC_SWEEP / NBINS) + ((TCACHE_GC_SWEEP / NBINS == 0) ? 0 : 1))
61
62#endif /* JEMALLOC_H_TYPES */
63/******************************************************************************/
64#ifdef JEMALLOC_H_STRUCTS
65
66typedef enum {
67	tcache_enabled_false   = 0, /* Enable cast to/from bool. */
68	tcache_enabled_true    = 1,
69	tcache_enabled_default = 2
70} tcache_enabled_t;
71
72/*
73 * Read-only information associated with each element of tcache_t's tbins array
74 * is stored separately, mainly to reduce memory usage.
75 */
76struct tcache_bin_info_s {
77	unsigned	ncached_max;	/* Upper limit on ncached. */
78};
79
80struct tcache_bin_s {
81	tcache_bin_stats_t tstats;
82	int		low_water;	/* Min # cached since last GC. */
83	unsigned	lg_fill_div;	/* Fill (ncached_max >> lg_fill_div). */
84	unsigned	ncached;	/* # of cached objects. */
85	void		**avail;	/* Stack of available objects. */
86};
87
88struct tcache_s {
89	ql_elm(tcache_t) link;		/* Used for aggregating stats. */
90	uint64_t	prof_accumbytes;/* Cleared after arena_prof_accum(). */
91	unsigned	ev_cnt;		/* Event count since incremental GC. */
92	index_t		next_gc_bin;	/* Next bin to GC. */
93	tcache_bin_t	tbins[1];	/* Dynamically sized. */
94	/*
95	 * The pointer stacks associated with tbins follow as a contiguous
96	 * array.  During tcache initialization, the avail pointer in each
97	 * element of tbins is initialized to point to the proper offset within
98	 * this array.
99	 */
100};
101
102/* Linkage for list of available (previously used) explicit tcache IDs. */
103struct tcaches_s {
104	union {
105		tcache_t	*tcache;
106		tcaches_t	*next;
107	};
108};
109
110#endif /* JEMALLOC_H_STRUCTS */
111/******************************************************************************/
112#ifdef JEMALLOC_H_EXTERNS
113
114extern bool	opt_tcache;
115extern ssize_t	opt_lg_tcache_max;
116
117extern tcache_bin_info_t	*tcache_bin_info;
118
119/*
120 * Number of tcache bins.  There are NBINS small-object bins, plus 0 or more
121 * large-object bins.
122 */
123extern size_t	nhbins;
124
125/* Maximum cached size class. */
126extern size_t	tcache_maxclass;
127
128/*
129 * Explicit tcaches, managed via the tcache.{create,flush,destroy} mallctls and
130 * usable via the MALLOCX_TCACHE() flag.  The automatic per thread tcaches are
131 * completely disjoint from this data structure.  tcaches starts off as a sparse
132 * array, so it has no physical memory footprint until individual pages are
133 * touched.  This allows the entire array to be allocated the first time an
134 * explicit tcache is created without a disproportionate impact on memory usage.
135 */
136extern tcaches_t	*tcaches;
137
138size_t	tcache_salloc(const void *ptr);
139void	tcache_event_hard(tsd_t *tsd, tcache_t *tcache);
140void	*tcache_alloc_small_hard(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
141    tcache_bin_t *tbin, index_t binind);
142void	tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
143    index_t binind, unsigned rem);
144void	tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, index_t binind,
145    unsigned rem, tcache_t *tcache);
146void	tcache_arena_associate(tcache_t *tcache, arena_t *arena);
147void	tcache_arena_reassociate(tcache_t *tcache, arena_t *oldarena,
148    arena_t *newarena);
149void	tcache_arena_dissociate(tcache_t *tcache, arena_t *arena);
150tcache_t *tcache_get_hard(tsd_t *tsd);
151tcache_t *tcache_create(tsd_t *tsd, arena_t *arena);
152void	tcache_cleanup(tsd_t *tsd);
153void	tcache_enabled_cleanup(tsd_t *tsd);
154void	tcache_stats_merge(tcache_t *tcache, arena_t *arena);
155bool	tcaches_create(tsd_t *tsd, unsigned *r_ind);
156void	tcaches_flush(tsd_t *tsd, unsigned ind);
157void	tcaches_destroy(tsd_t *tsd, unsigned ind);
158bool	tcache_boot(void);
159
160#endif /* JEMALLOC_H_EXTERNS */
161/******************************************************************************/
162#ifdef JEMALLOC_H_INLINES
163
164#ifndef JEMALLOC_ENABLE_INLINE
165void	tcache_event(tsd_t *tsd, tcache_t *tcache);
166void	tcache_flush(void);
167bool	tcache_enabled_get(void);
168tcache_t *tcache_get(tsd_t *tsd, bool create);
169void	tcache_enabled_set(bool enabled);
170void	*tcache_alloc_easy(tcache_bin_t *tbin);
171void	*tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
172    size_t size, bool zero);
173void	*tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
174    size_t size, bool zero);
175void	tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr,
176    index_t binind);
177void	tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr,
178    size_t size);
179tcache_t	*tcaches_get(tsd_t *tsd, unsigned ind);
180#endif
181
182#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_))
183JEMALLOC_INLINE void
184tcache_flush(void)
185{
186	tsd_t *tsd;
187
188	cassert(config_tcache);
189
190	tsd = tsd_fetch();
191	tcache_cleanup(tsd);
192}
193
194JEMALLOC_INLINE bool
195tcache_enabled_get(void)
196{
197	tsd_t *tsd;
198	tcache_enabled_t tcache_enabled;
199
200	cassert(config_tcache);
201
202	tsd = tsd_fetch();
203	tcache_enabled = tsd_tcache_enabled_get(tsd);
204	if (tcache_enabled == tcache_enabled_default) {
205		tcache_enabled = (tcache_enabled_t)opt_tcache;
206		tsd_tcache_enabled_set(tsd, tcache_enabled);
207	}
208
209	return ((bool)tcache_enabled);
210}
211
212JEMALLOC_INLINE void
213tcache_enabled_set(bool enabled)
214{
215	tsd_t *tsd;
216	tcache_enabled_t tcache_enabled;
217
218	cassert(config_tcache);
219
220	tsd = tsd_fetch();
221
222	tcache_enabled = (tcache_enabled_t)enabled;
223	tsd_tcache_enabled_set(tsd, tcache_enabled);
224
225	if (!enabled)
226		tcache_cleanup(tsd);
227}
228
229JEMALLOC_ALWAYS_INLINE tcache_t *
230tcache_get(tsd_t *tsd, bool create)
231{
232	tcache_t *tcache;
233
234	if (!config_tcache)
235		return (NULL);
236
237	tcache = tsd_tcache_get(tsd);
238	if (!create)
239		return (tcache);
240	if (unlikely(tcache == NULL) && tsd_nominal(tsd)) {
241		tcache = tcache_get_hard(tsd);
242		tsd_tcache_set(tsd, tcache);
243	}
244
245	return (tcache);
246}
247
248JEMALLOC_ALWAYS_INLINE void
249tcache_event(tsd_t *tsd, tcache_t *tcache)
250{
251
252	if (TCACHE_GC_INCR == 0)
253		return;
254
255	tcache->ev_cnt++;
256	assert(tcache->ev_cnt <= TCACHE_GC_INCR);
257	if (unlikely(tcache->ev_cnt == TCACHE_GC_INCR))
258		tcache_event_hard(tsd, tcache);
259}
260
261JEMALLOC_ALWAYS_INLINE void *
262tcache_alloc_easy(tcache_bin_t *tbin)
263{
264	void *ret;
265
266	if (unlikely(tbin->ncached == 0)) {
267		tbin->low_water = -1;
268		return (NULL);
269	}
270	tbin->ncached--;
271	if (unlikely((int)tbin->ncached < tbin->low_water))
272		tbin->low_water = tbin->ncached;
273	ret = tbin->avail[tbin->ncached];
274	return (ret);
275}
276
277JEMALLOC_ALWAYS_INLINE void *
278tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
279    bool zero)
280{
281	void *ret;
282	index_t binind;
283	size_t usize;
284	tcache_bin_t *tbin;
285
286	binind = size2index(size);
287	assert(binind < NBINS);
288	tbin = &tcache->tbins[binind];
289	usize = index2size(binind);
290	ret = tcache_alloc_easy(tbin);
291	if (unlikely(ret == NULL)) {
292		ret = tcache_alloc_small_hard(tsd, arena, tcache, tbin, binind);
293		if (ret == NULL)
294			return (NULL);
295	}
296	assert(tcache_salloc(ret) == usize);
297
298	if (likely(!zero)) {
299		if (config_fill) {
300			if (unlikely(opt_junk_alloc)) {
301				arena_alloc_junk_small(ret,
302				    &arena_bin_info[binind], false);
303			} else if (unlikely(opt_zero))
304				memset(ret, 0, usize);
305		}
306	} else {
307		if (config_fill && unlikely(opt_junk_alloc)) {
308			arena_alloc_junk_small(ret, &arena_bin_info[binind],
309			    true);
310		}
311		memset(ret, 0, usize);
312	}
313
314	if (config_stats)
315		tbin->tstats.nrequests++;
316	if (config_prof)
317		tcache->prof_accumbytes += usize;
318	tcache_event(tsd, tcache);
319	return (ret);
320}
321
322JEMALLOC_ALWAYS_INLINE void *
323tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
324    bool zero)
325{
326	void *ret;
327	index_t binind;
328	size_t usize;
329	tcache_bin_t *tbin;
330
331	binind = size2index(size);
332	usize = index2size(binind);
333	assert(usize <= tcache_maxclass);
334	assert(binind < nhbins);
335	tbin = &tcache->tbins[binind];
336	ret = tcache_alloc_easy(tbin);
337	if (unlikely(ret == NULL)) {
338		/*
339		 * Only allocate one large object at a time, because it's quite
340		 * expensive to create one and not use it.
341		 */
342		ret = arena_malloc_large(arena, usize, zero);
343		if (ret == NULL)
344			return (NULL);
345	} else {
346		if (config_prof && usize == LARGE_MINCLASS) {
347			arena_chunk_t *chunk =
348			    (arena_chunk_t *)CHUNK_ADDR2BASE(ret);
349			size_t pageind = (((uintptr_t)ret - (uintptr_t)chunk) >>
350			    LG_PAGE);
351			arena_mapbits_large_binind_set(chunk, pageind,
352			    BININD_INVALID);
353		}
354		if (likely(!zero)) {
355			if (config_fill) {
356				if (unlikely(opt_junk_alloc))
357					memset(ret, 0xa5, usize);
358				else if (unlikely(opt_zero))
359					memset(ret, 0, usize);
360			}
361		} else
362			memset(ret, 0, usize);
363
364		if (config_stats)
365			tbin->tstats.nrequests++;
366		if (config_prof)
367			tcache->prof_accumbytes += usize;
368	}
369
370	tcache_event(tsd, tcache);
371	return (ret);
372}
373
374JEMALLOC_ALWAYS_INLINE void
375tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, index_t binind)
376{
377	tcache_bin_t *tbin;
378	tcache_bin_info_t *tbin_info;
379
380	assert(tcache_salloc(ptr) <= SMALL_MAXCLASS);
381
382	if (config_fill && unlikely(opt_junk_free))
383		arena_dalloc_junk_small(ptr, &arena_bin_info[binind]);
384
385	tbin = &tcache->tbins[binind];
386	tbin_info = &tcache_bin_info[binind];
387	if (unlikely(tbin->ncached == tbin_info->ncached_max)) {
388		tcache_bin_flush_small(tsd, tcache, tbin, binind,
389		    (tbin_info->ncached_max >> 1));
390	}
391	assert(tbin->ncached < tbin_info->ncached_max);
392	tbin->avail[tbin->ncached] = ptr;
393	tbin->ncached++;
394
395	tcache_event(tsd, tcache);
396}
397
398JEMALLOC_ALWAYS_INLINE void
399tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, size_t size)
400{
401	index_t binind;
402	tcache_bin_t *tbin;
403	tcache_bin_info_t *tbin_info;
404
405	assert((size & PAGE_MASK) == 0);
406	assert(tcache_salloc(ptr) > SMALL_MAXCLASS);
407	assert(tcache_salloc(ptr) <= tcache_maxclass);
408
409	binind = size2index(size);
410
411	if (config_fill && unlikely(opt_junk_free))
412		arena_dalloc_junk_large(ptr, size);
413
414	tbin = &tcache->tbins[binind];
415	tbin_info = &tcache_bin_info[binind];
416	if (unlikely(tbin->ncached == tbin_info->ncached_max)) {
417		tcache_bin_flush_large(tsd, tbin, binind,
418		    (tbin_info->ncached_max >> 1), tcache);
419	}
420	assert(tbin->ncached < tbin_info->ncached_max);
421	tbin->avail[tbin->ncached] = ptr;
422	tbin->ncached++;
423
424	tcache_event(tsd, tcache);
425}
426
427JEMALLOC_ALWAYS_INLINE tcache_t *
428tcaches_get(tsd_t *tsd, unsigned ind)
429{
430	tcaches_t *elm = &tcaches[ind];
431	if (unlikely(elm->tcache == NULL))
432		elm->tcache = tcache_create(tsd, arena_choose(tsd, NULL));
433	return (elm->tcache);
434}
435#endif
436
437#endif /* JEMALLOC_H_INLINES */
438/******************************************************************************/
439