1e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define JEMALLOC_BASE_C_ 2376b1529a383c39adf4674baf6db83a5e63f97acJason Evans#include "jemalloc/internal/jemalloc_internal.h" 3e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 44201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans/******************************************************************************/ 54201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans/* Data. */ 6e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 74e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansstatic malloc_mutex_t base_mtx; 8e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 9e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 10e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Current pages that are being used for internal memory allocations. These 11e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * pages are carved up in cacheline-size quanta, so that there is no chance of 12e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * false cache line sharing. 13e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 14e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void *base_pages; 15e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void *base_next_addr; 16e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void *base_past_addr; /* Addr immediately past base_pages. */ 17e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic extent_node_t *base_nodes; 18e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 194201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans/******************************************************************************/ 20e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 21e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 224201af05425b69ee37ffca437aca0cdd604d1e51Jason Evansbase_pages_alloc(size_t minsize) 23e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 24e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t csize; 25e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 26e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(minsize != 0); 274201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans csize = CHUNK_CEILING(minsize); 2812141150fdbda57651a53ae2fe0edaea4891d814Jason Evans base_pages = chunk_alloc_base(csize); 29e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (base_pages == NULL) 30e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 31e476f8a161d445211fd6e54fe370275196e66bcbJason Evans base_next_addr = base_pages; 32e476f8a161d445211fd6e54fe370275196e66bcbJason Evans base_past_addr = (void *)((uintptr_t)base_pages + csize); 33e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 34e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 35e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 36e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 37e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 38e476f8a161d445211fd6e54fe370275196e66bcbJason Evansbase_alloc(size_t size) 39e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 40e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 41e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t csize; 42e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 43e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Round size up to nearest multiple of the cacheline size. */ 44e476f8a161d445211fd6e54fe370275196e66bcbJason Evans csize = CACHELINE_CEILING(size); 45e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 46e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&base_mtx); 47e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Make sure there's enough space for the allocation. */ 48e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((uintptr_t)base_next_addr + csize > (uintptr_t)base_past_addr) { 49e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (base_pages_alloc(csize)) { 50e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&base_mtx); 51e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 52e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 53e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 54e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Allocate. */ 55e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = base_next_addr; 56e476f8a161d445211fd6e54fe370275196e66bcbJason Evans base_next_addr = (void *)((uintptr_t)base_next_addr + csize); 57e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&base_mtx); 58bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, csize); 59e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 60e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 61e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 62e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 6341b6afb834b1f5250223678c52bd4f013d4234f6Jason Evansvoid * 6441b6afb834b1f5250223678c52bd4f013d4234f6Jason Evansbase_calloc(size_t number, size_t size) 6541b6afb834b1f5250223678c52bd4f013d4234f6Jason Evans{ 6641b6afb834b1f5250223678c52bd4f013d4234f6Jason Evans void *ret = base_alloc(number * size); 6741b6afb834b1f5250223678c52bd4f013d4234f6Jason Evans 6841b6afb834b1f5250223678c52bd4f013d4234f6Jason Evans if (ret != NULL) 6941b6afb834b1f5250223678c52bd4f013d4234f6Jason Evans memset(ret, 0, number * size); 7041b6afb834b1f5250223678c52bd4f013d4234f6Jason Evans 7141b6afb834b1f5250223678c52bd4f013d4234f6Jason Evans return (ret); 7241b6afb834b1f5250223678c52bd4f013d4234f6Jason Evans} 7341b6afb834b1f5250223678c52bd4f013d4234f6Jason Evans 74e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextent_node_t * 75e476f8a161d445211fd6e54fe370275196e66bcbJason Evansbase_node_alloc(void) 76e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 77e476f8a161d445211fd6e54fe370275196e66bcbJason Evans extent_node_t *ret; 78e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 79e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&base_mtx); 80e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (base_nodes != NULL) { 81e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = base_nodes; 82e476f8a161d445211fd6e54fe370275196e66bcbJason Evans base_nodes = *(extent_node_t **)ret; 83e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&base_mtx); 84bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, 85bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans sizeof(extent_node_t)); 86e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 87e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&base_mtx); 88e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (extent_node_t *)base_alloc(sizeof(extent_node_t)); 89e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 90e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 91e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 92e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 93e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 94e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 9512141150fdbda57651a53ae2fe0edaea4891d814Jason Evansbase_node_dalloc(extent_node_t *node) 96e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 97e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 98bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t)); 99e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&base_mtx); 100e476f8a161d445211fd6e54fe370275196e66bcbJason Evans *(extent_node_t **)node = base_nodes; 101e476f8a161d445211fd6e54fe370275196e66bcbJason Evans base_nodes = node; 102e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&base_mtx); 103e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 104e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 105e476f8a161d445211fd6e54fe370275196e66bcbJason Evansbool 106e476f8a161d445211fd6e54fe370275196e66bcbJason Evansbase_boot(void) 107e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 108e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 109e476f8a161d445211fd6e54fe370275196e66bcbJason Evans base_nodes = NULL; 110e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (malloc_mutex_init(&base_mtx)) 111e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 112e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 113e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 114e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1154e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 1164e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid 1174e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansbase_prefork(void) 1184e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans{ 1194e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 1204e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_prefork(&base_mtx); 1214e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans} 1224e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 1234e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid 1244e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansbase_postfork_parent(void) 1254e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans{ 1264e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 1274e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_postfork_parent(&base_mtx); 1284e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans} 1294e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 1304e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid 1314e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansbase_postfork_child(void) 1324e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans{ 1334e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 1344e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_postfork_child(&base_mtx); 1354e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans} 136