tcache.c revision dafde14e08ddfda747aabb2045b350848b601b2e 
1#define	JEMALLOC_TCACHE_C_
2#include "jemalloc/internal/jemalloc_internal.h"
3#ifdef JEMALLOC_TCACHE
4/******************************************************************************/
5/* Data. */
6
7bool	opt_tcache = true;
8ssize_t	opt_lg_tcache_maxclass = LG_TCACHE_MAXCLASS_DEFAULT;
9ssize_t	opt_lg_tcache_gc_sweep = LG_TCACHE_GC_SWEEP_DEFAULT;
10
11/* Map of thread-specific caches. */
12__thread tcache_t	*tcache_tls JEMALLOC_ATTR(tls_model("initial-exec"));
13
14/*
15 * Same contents as tcache, but initialized such that the TSD destructor is
16 * called when a thread exits, so that the cache can be cleaned up.
17 */
18static pthread_key_t		tcache_tsd;
19
20size_t				nhbins;
21size_t				tcache_maxclass;
22unsigned			tcache_gc_incr;
23
24/******************************************************************************/
25/* Function prototypes for non-inline static functions. */
26
27static void	tcache_thread_cleanup(void *arg);
28
29/******************************************************************************/
30
31void *
32tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind)
33{
34	void *ret;
35
36	arena_tcache_fill_small(tcache->arena, tbin, binind
37#ifdef JEMALLOC_PROF
38	    , tcache->prof_accumbytes
39#endif
40	    );
41#ifdef JEMALLOC_PROF
42	tcache->prof_accumbytes = 0;
43#endif
44	ret = tcache_alloc_easy(tbin);
45
46	return (ret);
47}
48
49void
50tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem
51#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
52    , tcache_t *tcache
53#endif
54    )
55{
56	void *flush, *deferred, *ptr;
57	unsigned i, nflush, ndeferred;
58
59	assert(binind < nbins);
60	assert(rem <= tbin->ncached);
61
62	for (flush = tbin->avail, nflush = tbin->ncached - rem; flush != NULL;
63	    flush = deferred, nflush = ndeferred) {
64		/* Lock the arena bin associated with the first object. */
65		arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(flush);
66		arena_t *arena = chunk->arena;
67		arena_bin_t *bin = &arena->bins[binind];
68
69#ifdef JEMALLOC_PROF
70		if (arena == tcache->arena) {
71			malloc_mutex_lock(&arena->lock);
72			arena_prof_accum(arena, tcache->prof_accumbytes);
73			malloc_mutex_unlock(&arena->lock);
74			tcache->prof_accumbytes = 0;
75		}
76#endif
77
78		malloc_mutex_lock(&bin->lock);
79#ifdef JEMALLOC_STATS
80		if (arena == tcache->arena) {
81			bin->stats.nflushes++;
82			bin->stats.nrequests += tbin->tstats.nrequests;
83			tbin->tstats.nrequests = 0;
84		}
85#endif
86		deferred = NULL;
87		ndeferred = 0;
88		for (i = 0; i < nflush; i++) {
89			ptr = flush;
90			assert(ptr != NULL);
91			flush = *(void **)ptr;
92			chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
93			if (chunk->arena == arena) {
94				size_t pageind = (((uintptr_t)ptr -
95				    (uintptr_t)chunk) >> PAGE_SHIFT);
96				arena_chunk_map_t *mapelm =
97				    &chunk->map[pageind];
98				arena_dalloc_bin(arena, chunk, ptr, mapelm);
99			} else {
100				/*
101				 * This object was allocated via a different
102				 * arena bin than the one that is currently
103				 * locked.  Stash the object, so that it can be
104				 * handled in a future pass.
105				 */
106				*(void **)ptr = deferred;
107				deferred = ptr;
108				ndeferred++;
109			}
110		}
111		malloc_mutex_unlock(&bin->lock);
112
113		if (flush != NULL) {
114			/*
115			 * This was the first pass, and rem cached objects
116			 * remain.
117			 */
118			tbin->avail = flush;
119		}
120	}
121
122	tbin->ncached = rem;
123	if (tbin->ncached < tbin->low_water)
124		tbin->low_water = tbin->ncached;
125}
126
127void
128tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem
129#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
130    , tcache_t *tcache
131#endif
132    )
133{
134	void *flush, *deferred, *ptr;
135	unsigned i, nflush, ndeferred;
136
137	assert(binind < nhbins);
138	assert(rem <= tbin->ncached);
139
140	for (flush = tbin->avail, nflush = tbin->ncached - rem; flush != NULL;
141	    flush = deferred, nflush = ndeferred) {
142		/* Lock the arena associated with the first object. */
143		arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(flush);
144		arena_t *arena = chunk->arena;
145
146		malloc_mutex_lock(&arena->lock);
147#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS))
148		if (arena == tcache->arena) {
149#endif
150#ifdef JEMALLOC_PROF
151			arena_prof_accum(arena, tcache->prof_accumbytes);
152			tcache->prof_accumbytes = 0;
153#endif
154#ifdef JEMALLOC_STATS
155			arena->stats.nrequests_large += tbin->tstats.nrequests;
156			arena->stats.lstats[binind - nbins].nrequests +=
157			    tbin->tstats.nrequests;
158			tbin->tstats.nrequests = 0;
159#endif
160#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS))
161		}
162#endif
163		deferred = NULL;
164		ndeferred = 0;
165		for (i = 0; i < nflush; i++) {
166			ptr = flush;
167			assert(ptr != NULL);
168			flush = *(void **)ptr;
169			chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
170			if (chunk->arena == arena)
171				arena_dalloc_large(arena, chunk, ptr);
172			else {
173				/*
174				 * This object was allocated via a different
175				 * arena than the one that is currently locked.
176				 * Stash the object, so that it can be handled
177				 * in a future pass.
178				 */
179				*(void **)ptr = deferred;
180				deferred = ptr;
181				ndeferred++;
182			}
183		}
184		malloc_mutex_unlock(&arena->lock);
185
186		if (flush != NULL) {
187			/*
188			 * This was the first pass, and rem cached objects
189			 * remain.
190			 */
191			tbin->avail = flush;
192		}
193	}
194
195	tbin->ncached = rem;
196	if (tbin->ncached < tbin->low_water)
197		tbin->low_water = tbin->ncached;
198}
199
200tcache_t *
201tcache_create(arena_t *arena)
202{
203	tcache_t *tcache;
204	size_t size;
205	unsigned i;
206
207	size = sizeof(tcache_t) + (sizeof(tcache_bin_t) * (nhbins - 1));
208	/*
209	 * Round up to the nearest multiple of the cacheline size, in order to
210	 * avoid the possibility of false cacheline sharing.
211	 *
212	 * That this works relies on the same logic as in ipalloc().
213	 */
214	size = (size + CACHELINE_MASK) & (-CACHELINE);
215
216	if (size <= small_maxclass)
217		tcache = (tcache_t *)arena_malloc_small(arena, size, true);
218	else
219		tcache = (tcache_t *)icalloc(size);
220
221	if (tcache == NULL)
222		return (NULL);
223
224#ifdef JEMALLOC_STATS
225	/* Link into list of extant tcaches. */
226	malloc_mutex_lock(&arena->lock);
227	ql_elm_new(tcache, link);
228	ql_tail_insert(&arena->tcache_ql, tcache, link);
229	malloc_mutex_unlock(&arena->lock);
230#endif
231
232	tcache->arena = arena;
233	assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0);
234	for (i = 0; i < nbins; i++) {
235		if ((arena->bins[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MAX) {
236			tcache->tbins[i].ncached_max = (arena->bins[i].nregs <<
237			    1);
238		} else
239			tcache->tbins[i].ncached_max = TCACHE_NSLOTS_SMALL_MAX;
240	}
241	for (; i < nhbins; i++)
242		tcache->tbins[i].ncached_max = TCACHE_NSLOTS_LARGE;
243
244	tcache_tls = tcache;
245	pthread_setspecific(tcache_tsd, tcache);
246
247	return (tcache);
248}
249
250void
251tcache_destroy(tcache_t *tcache)
252{
253	unsigned i;
254
255#ifdef JEMALLOC_STATS
256	/* Unlink from list of extant tcaches. */
257	malloc_mutex_lock(&tcache->arena->lock);
258	ql_remove(&tcache->arena->tcache_ql, tcache, link);
259	malloc_mutex_unlock(&tcache->arena->lock);
260	tcache_stats_merge(tcache, tcache->arena);
261#endif
262
263	for (i = 0; i < nbins; i++) {
264		tcache_bin_t *tbin = &tcache->tbins[i];
265		tcache_bin_flush_small(tbin, i, 0
266#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
267		    , tcache
268#endif
269		    );
270
271#ifdef JEMALLOC_STATS
272		if (tbin->tstats.nrequests != 0) {
273			arena_t *arena = tcache->arena;
274			arena_bin_t *bin = &arena->bins[i];
275			malloc_mutex_lock(&bin->lock);
276			bin->stats.nrequests += tbin->tstats.nrequests;
277			malloc_mutex_unlock(&bin->lock);
278		}
279#endif
280	}
281
282	for (; i < nhbins; i++) {
283		tcache_bin_t *tbin = &tcache->tbins[i];
284		tcache_bin_flush_large(tbin, i, 0
285#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
286		    , tcache
287#endif
288		    );
289
290#ifdef JEMALLOC_STATS
291		if (tbin->tstats.nrequests != 0) {
292			arena_t *arena = tcache->arena;
293			malloc_mutex_lock(&arena->lock);
294			arena->stats.nrequests_large += tbin->tstats.nrequests;
295			arena->stats.lstats[i - nbins].nrequests +=
296			    tbin->tstats.nrequests;
297			malloc_mutex_unlock(&arena->lock);
298		}
299#endif
300	}
301
302#ifdef JEMALLOC_PROF
303	if (tcache->prof_accumbytes > 0) {
304		malloc_mutex_lock(&tcache->arena->lock);
305		arena_prof_accum(tcache->arena, tcache->prof_accumbytes);
306		malloc_mutex_unlock(&tcache->arena->lock);
307	}
308#endif
309
310	if (arena_salloc(tcache) <= small_maxclass) {
311		arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache);
312		arena_t *arena = chunk->arena;
313		size_t pageind = (((uintptr_t)tcache - (uintptr_t)chunk) >>
314		    PAGE_SHIFT);
315		arena_chunk_map_t *mapelm = &chunk->map[pageind];
316		arena_run_t *run = (arena_run_t *)((uintptr_t)chunk +
317		    (uintptr_t)((pageind - ((mapelm->bits & CHUNK_MAP_PG_MASK)
318		    >> CHUNK_MAP_PG_SHIFT)) << PAGE_SHIFT));
319		arena_bin_t *bin = run->bin;
320
321		malloc_mutex_lock(&bin->lock);
322		arena_dalloc_bin(arena, chunk, tcache, mapelm);
323		malloc_mutex_unlock(&bin->lock);
324	} else
325		idalloc(tcache);
326}
327
328static void
329tcache_thread_cleanup(void *arg)
330{
331	tcache_t *tcache = (tcache_t *)arg;
332
333	assert(tcache == tcache_tls);
334	if (tcache != NULL) {
335		assert(tcache != (void *)(uintptr_t)1);
336		tcache_destroy(tcache);
337		tcache_tls = (void *)(uintptr_t)1;
338	}
339}
340
341#ifdef JEMALLOC_STATS
342void
343tcache_stats_merge(tcache_t *tcache, arena_t *arena)
344{
345	unsigned i;
346
347	/* Merge and reset tcache stats. */
348	for (i = 0; i < nbins; i++) {
349		arena_bin_t *bin = &arena->bins[i];
350		tcache_bin_t *tbin = &tcache->tbins[i];
351		malloc_mutex_lock(&bin->lock);
352		bin->stats.nrequests += tbin->tstats.nrequests;
353		malloc_mutex_unlock(&bin->lock);
354		tbin->tstats.nrequests = 0;
355	}
356
357	for (; i < nhbins; i++) {
358		malloc_large_stats_t *lstats = &arena->stats.lstats[i - nbins];
359		tcache_bin_t *tbin = &tcache->tbins[i];
360		arena->stats.nrequests_large += tbin->tstats.nrequests;
361		lstats->nrequests += tbin->tstats.nrequests;
362		tbin->tstats.nrequests = 0;
363	}
364}
365#endif
366
367void
368tcache_boot(void)
369{
370
371	if (opt_tcache) {
372		/*
373		 * If necessary, clamp opt_lg_tcache_maxclass, now that
374		 * small_maxclass and arena_maxclass are known.
375		 */
376		if (opt_lg_tcache_maxclass < 0 || (1U <<
377		    opt_lg_tcache_maxclass) < small_maxclass)
378			tcache_maxclass = small_maxclass;
379		else if ((1U << opt_lg_tcache_maxclass) > arena_maxclass)
380			tcache_maxclass = arena_maxclass;
381		else
382			tcache_maxclass = (1U << opt_lg_tcache_maxclass);
383
384		nhbins = nbins + (tcache_maxclass >> PAGE_SHIFT);
385
386		/* Compute incremental GC event threshold. */
387		if (opt_lg_tcache_gc_sweep >= 0) {
388			tcache_gc_incr = ((1U << opt_lg_tcache_gc_sweep) /
389			    nbins) + (((1U << opt_lg_tcache_gc_sweep) % nbins ==
390			    0) ? 0 : 1);
391		} else
392			tcache_gc_incr = 0;
393
394		if (pthread_key_create(&tcache_tsd, tcache_thread_cleanup) !=
395		    0) {
396			malloc_write(
397			    "<jemalloc>: Error in pthread_key_create()\n");
398			abort();
399		}
400	}
401}
402/******************************************************************************/
403#endif /* JEMALLOC_TCACHE */
404