1bbe29d374d0fa5f4684621f16c099294e56c26efJason Evans#define JEMALLOC_QUARANTINE_C_ 2122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans#include "jemalloc/internal/jemalloc_internal.h" 3122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 4577dd84660351c727a62db99b308e35d9da224a1Jason Evans/* 5577dd84660351c727a62db99b308e35d9da224a1Jason Evans * quarantine pointers close to NULL are used to encode state information that 6577dd84660351c727a62db99b308e35d9da224a1Jason Evans * is used for cleaning up during thread shutdown. 7577dd84660351c727a62db99b308e35d9da224a1Jason Evans */ 8577dd84660351c727a62db99b308e35d9da224a1Jason Evans#define QUARANTINE_STATE_REINCARNATED ((quarantine_t *)(uintptr_t)1) 9577dd84660351c727a62db99b308e35d9da224a1Jason Evans#define QUARANTINE_STATE_PURGATORY ((quarantine_t *)(uintptr_t)2) 10577dd84660351c727a62db99b308e35d9da224a1Jason Evans#define QUARANTINE_STATE_MAX QUARANTINE_STATE_PURGATORY 11577dd84660351c727a62db99b308e35d9da224a1Jason Evans 12122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans/******************************************************************************/ 13122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans/* Data. */ 14122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 15bbe29d374d0fa5f4684621f16c099294e56c26efJason Evansmalloc_tsd_data(, quarantine, quarantine_t *, NULL) 16122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 17122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans/******************************************************************************/ 18122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans/* Function prototypes for non-inline static functions. */ 19122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 20122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansstatic quarantine_t *quarantine_grow(quarantine_t *quarantine); 21d0e942e4669b8600b0bd7e5ae132ae26d10a40edJason Evansstatic void quarantine_drain_one(quarantine_t *quarantine); 22122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansstatic void quarantine_drain(quarantine_t *quarantine, size_t upper_bound); 23122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 24122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans/******************************************************************************/ 25122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 26bbe29d374d0fa5f4684621f16c099294e56c26efJason Evansquarantine_t * 27122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansquarantine_init(size_t lg_maxobjs) 28122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans{ 29122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans quarantine_t *quarantine; 30122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 31122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans quarantine = (quarantine_t *)imalloc(offsetof(quarantine_t, objs) + 329cd351d147d1e79bff6b89586f168e81c0be034eJason Evans ((ZU(1) << lg_maxobjs) * sizeof(quarantine_obj_t))); 33122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (quarantine == NULL) 34122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans return (NULL); 35122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans quarantine->curbytes = 0; 36122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans quarantine->curobjs = 0; 37122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans quarantine->first = 0; 38122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans quarantine->lg_maxobjs = lg_maxobjs; 39122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 40122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans quarantine_tsd_set(&quarantine); 41122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 42122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans return (quarantine); 43122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans} 44122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 45122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansstatic quarantine_t * 46122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansquarantine_grow(quarantine_t *quarantine) 47122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans{ 48122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans quarantine_t *ret; 49122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 50122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans ret = quarantine_init(quarantine->lg_maxobjs + 1); 51d0e942e4669b8600b0bd7e5ae132ae26d10a40edJason Evans if (ret == NULL) { 52d0e942e4669b8600b0bd7e5ae132ae26d10a40edJason Evans quarantine_drain_one(quarantine); 53122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans return (quarantine); 54d0e942e4669b8600b0bd7e5ae132ae26d10a40edJason Evans } 55122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 56122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans ret->curbytes = quarantine->curbytes; 577e060397a38e301d7d3317fbf138f342c2bbd1b9Jason Evans ret->curobjs = quarantine->curobjs; 587e060397a38e301d7d3317fbf138f342c2bbd1b9Jason Evans if (quarantine->first + quarantine->curobjs <= (ZU(1) << 59122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans quarantine->lg_maxobjs)) { 60122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans /* objs ring buffer data are contiguous. */ 61122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans memcpy(ret->objs, &quarantine->objs[quarantine->first], 629cd351d147d1e79bff6b89586f168e81c0be034eJason Evans quarantine->curobjs * sizeof(quarantine_obj_t)); 63122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } else { 64122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans /* objs ring buffer data wrap around. */ 657e060397a38e301d7d3317fbf138f342c2bbd1b9Jason Evans size_t ncopy_a = (ZU(1) << quarantine->lg_maxobjs) - 66122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans quarantine->first; 677e060397a38e301d7d3317fbf138f342c2bbd1b9Jason Evans size_t ncopy_b = quarantine->curobjs - ncopy_a; 687e060397a38e301d7d3317fbf138f342c2bbd1b9Jason Evans 697e060397a38e301d7d3317fbf138f342c2bbd1b9Jason Evans memcpy(ret->objs, &quarantine->objs[quarantine->first], ncopy_a 707e060397a38e301d7d3317fbf138f342c2bbd1b9Jason Evans * sizeof(quarantine_obj_t)); 717e060397a38e301d7d3317fbf138f342c2bbd1b9Jason Evans memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b * 729cd351d147d1e79bff6b89586f168e81c0be034eJason Evans sizeof(quarantine_obj_t)); 73122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 74d0e942e4669b8600b0bd7e5ae132ae26d10a40edJason Evans idalloc(quarantine); 75122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 76122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans return (ret); 77122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans} 78122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 79122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansstatic void 80d0e942e4669b8600b0bd7e5ae132ae26d10a40edJason Evansquarantine_drain_one(quarantine_t *quarantine) 81d0e942e4669b8600b0bd7e5ae132ae26d10a40edJason Evans{ 82d0e942e4669b8600b0bd7e5ae132ae26d10a40edJason Evans quarantine_obj_t *obj = &quarantine->objs[quarantine->first]; 83d0e942e4669b8600b0bd7e5ae132ae26d10a40edJason Evans assert(obj->usize == isalloc(obj->ptr, config_prof)); 84d0e942e4669b8600b0bd7e5ae132ae26d10a40edJason Evans idalloc(obj->ptr); 85d0e942e4669b8600b0bd7e5ae132ae26d10a40edJason Evans quarantine->curbytes -= obj->usize; 86d0e942e4669b8600b0bd7e5ae132ae26d10a40edJason Evans quarantine->curobjs--; 87d0e942e4669b8600b0bd7e5ae132ae26d10a40edJason Evans quarantine->first = (quarantine->first + 1) & ((ZU(1) << 88d0e942e4669b8600b0bd7e5ae132ae26d10a40edJason Evans quarantine->lg_maxobjs) - 1); 89d0e942e4669b8600b0bd7e5ae132ae26d10a40edJason Evans} 90d0e942e4669b8600b0bd7e5ae132ae26d10a40edJason Evans 91d0e942e4669b8600b0bd7e5ae132ae26d10a40edJason Evansstatic void 92122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansquarantine_drain(quarantine_t *quarantine, size_t upper_bound) 93122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans{ 94122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 95d0e942e4669b8600b0bd7e5ae132ae26d10a40edJason Evans while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0) 96d0e942e4669b8600b0bd7e5ae132ae26d10a40edJason Evans quarantine_drain_one(quarantine); 97122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans} 98122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 99122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansvoid 100122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansquarantine(void *ptr) 101122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans{ 102122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans quarantine_t *quarantine; 103122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t usize = isalloc(ptr, config_prof); 104122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 10578f7352259768f670f8e1f9b000388dd32b62493Jason Evans cassert(config_fill); 106122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(opt_quarantine); 107122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 108122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans quarantine = *quarantine_tsd_get(); 109577dd84660351c727a62db99b308e35d9da224a1Jason Evans if ((uintptr_t)quarantine <= (uintptr_t)QUARANTINE_STATE_MAX) { 110bbe29d374d0fa5f4684621f16c099294e56c26efJason Evans if (quarantine == QUARANTINE_STATE_PURGATORY) { 111bbe29d374d0fa5f4684621f16c099294e56c26efJason Evans /* 112bbe29d374d0fa5f4684621f16c099294e56c26efJason Evans * Make a note that quarantine() was called after 113bbe29d374d0fa5f4684621f16c099294e56c26efJason Evans * quarantine_cleanup() was called. 114bbe29d374d0fa5f4684621f16c099294e56c26efJason Evans */ 115bbe29d374d0fa5f4684621f16c099294e56c26efJason Evans quarantine = QUARANTINE_STATE_REINCARNATED; 116bbe29d374d0fa5f4684621f16c099294e56c26efJason Evans quarantine_tsd_set(&quarantine); 117577dd84660351c727a62db99b308e35d9da224a1Jason Evans } 118bbe29d374d0fa5f4684621f16c099294e56c26efJason Evans idalloc(ptr); 119bbe29d374d0fa5f4684621f16c099294e56c26efJason Evans return; 120122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 121122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans /* 122122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * Drain one or more objects if the quarantine size limit would be 123122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * exceeded by appending ptr. 124122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans */ 125122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (quarantine->curbytes + usize > opt_quarantine) { 126122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t upper_bound = (opt_quarantine >= usize) ? opt_quarantine 127122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans - usize : 0; 128122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans quarantine_drain(quarantine, upper_bound); 129122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 130122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans /* Grow the quarantine ring buffer if it's full. */ 131122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (quarantine->curobjs == (ZU(1) << quarantine->lg_maxobjs)) 132122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans quarantine = quarantine_grow(quarantine); 133122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans /* quarantine_grow() must free a slot if it fails to grow. */ 134122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(quarantine->curobjs < (ZU(1) << quarantine->lg_maxobjs)); 135122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans /* Append ptr if its size doesn't exceed the quarantine size. */ 136122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (quarantine->curbytes + usize <= opt_quarantine) { 137122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t offset = (quarantine->first + quarantine->curobjs) & 138122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans ((ZU(1) << quarantine->lg_maxobjs) - 1); 1399cd351d147d1e79bff6b89586f168e81c0be034eJason Evans quarantine_obj_t *obj = &quarantine->objs[offset]; 1409cd351d147d1e79bff6b89586f168e81c0be034eJason Evans obj->ptr = ptr; 1419cd351d147d1e79bff6b89586f168e81c0be034eJason Evans obj->usize = usize; 142122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans quarantine->curbytes += usize; 143122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans quarantine->curobjs++; 1440d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans if (config_fill && opt_junk) { 1450d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans /* 1460d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans * Only do redzone validation if Valgrind isn't in 1470d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans * operation. 1480d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans */ 149ecd3e59ca351d7111ec72a327fe0c009f2aa69a0Jason Evans if ((config_valgrind == false || in_valgrind == false) 1500d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans && usize <= SMALL_MAXCLASS) 1510d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans arena_quarantine_junk_small(ptr, usize); 1520d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans else 1530d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans memset(ptr, 0x5a, usize); 1540d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans } 155122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } else { 156122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(quarantine->curbytes == 0); 157122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans idalloc(ptr); 158122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 159122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans} 160122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 161bbe29d374d0fa5f4684621f16c099294e56c26efJason Evansvoid 162122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansquarantine_cleanup(void *arg) 163122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans{ 164122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans quarantine_t *quarantine = *(quarantine_t **)arg; 165122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 166577dd84660351c727a62db99b308e35d9da224a1Jason Evans if (quarantine == QUARANTINE_STATE_REINCARNATED) { 167577dd84660351c727a62db99b308e35d9da224a1Jason Evans /* 168577dd84660351c727a62db99b308e35d9da224a1Jason Evans * Another destructor deallocated memory after this destructor 169577dd84660351c727a62db99b308e35d9da224a1Jason Evans * was called. Reset quarantine to QUARANTINE_STATE_PURGATORY 170577dd84660351c727a62db99b308e35d9da224a1Jason Evans * in order to receive another callback. 171577dd84660351c727a62db99b308e35d9da224a1Jason Evans */ 172577dd84660351c727a62db99b308e35d9da224a1Jason Evans quarantine = QUARANTINE_STATE_PURGATORY; 173577dd84660351c727a62db99b308e35d9da224a1Jason Evans quarantine_tsd_set(&quarantine); 174577dd84660351c727a62db99b308e35d9da224a1Jason Evans } else if (quarantine == QUARANTINE_STATE_PURGATORY) { 175577dd84660351c727a62db99b308e35d9da224a1Jason Evans /* 176577dd84660351c727a62db99b308e35d9da224a1Jason Evans * The previous time this destructor was called, we set the key 177577dd84660351c727a62db99b308e35d9da224a1Jason Evans * to QUARANTINE_STATE_PURGATORY so that other destructors 178577dd84660351c727a62db99b308e35d9da224a1Jason Evans * wouldn't cause re-creation of the quarantine. This time, do 179577dd84660351c727a62db99b308e35d9da224a1Jason Evans * nothing, so that the destructor will not be called again. 180577dd84660351c727a62db99b308e35d9da224a1Jason Evans */ 181577dd84660351c727a62db99b308e35d9da224a1Jason Evans } else if (quarantine != NULL) { 182122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans quarantine_drain(quarantine, 0); 183122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans idalloc(quarantine); 184577dd84660351c727a62db99b308e35d9da224a1Jason Evans quarantine = QUARANTINE_STATE_PURGATORY; 185577dd84660351c727a62db99b308e35d9da224a1Jason Evans quarantine_tsd_set(&quarantine); 186122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 187122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans} 188122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 189122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansbool 190122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansquarantine_boot(void) 191122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans{ 192122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 19378f7352259768f670f8e1f9b000388dd32b62493Jason Evans cassert(config_fill); 194122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 195122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (quarantine_tsd_boot()) 196122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans return (true); 197122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 198122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans return (false); 199122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans} 200