1#define JEMALLOC_BASE_C_ 2#include "jemalloc/internal/jemalloc_internal.h" 3 4/******************************************************************************/ 5/* Data. */ 6 7static malloc_mutex_t base_mtx; 8static extent_tree_t base_avail_szad; 9static extent_node_t *base_nodes; 10static size_t base_allocated; 11static size_t base_resident; 12static size_t base_mapped; 13 14/******************************************************************************/ 15 16/* base_mtx must be held. */ 17static extent_node_t * 18base_node_try_alloc(void) 19{ 20 extent_node_t *node; 21 22 if (base_nodes == NULL) 23 return (NULL); 24 node = base_nodes; 25 base_nodes = *(extent_node_t **)node; 26 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t)); 27 return (node); 28} 29 30/* base_mtx must be held. */ 31static void 32base_node_dalloc(extent_node_t *node) 33{ 34 35 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t)); 36 *(extent_node_t **)node = base_nodes; 37 base_nodes = node; 38} 39 40/* base_mtx must be held. */ 41static extent_node_t * 42base_chunk_alloc(size_t minsize) 43{ 44 extent_node_t *node; 45 size_t csize, nsize; 46 void *addr; 47 48 assert(minsize != 0); 49 node = base_node_try_alloc(); 50 /* Allocate enough space to also carve a node out if necessary. */ 51 nsize = (node == NULL) ? CACHELINE_CEILING(sizeof(extent_node_t)) : 0; 52 csize = CHUNK_CEILING(minsize + nsize); 53 addr = chunk_alloc_base(csize); 54 if (addr == NULL) { 55 if (node != NULL) 56 base_node_dalloc(node); 57 return (NULL); 58 } 59 base_mapped += csize; 60 if (node == NULL) { 61 node = (extent_node_t *)addr; 62 addr = (void *)((uintptr_t)addr + nsize); 63 csize -= nsize; 64 if (config_stats) { 65 base_allocated += nsize; 66 base_resident += PAGE_CEILING(nsize); 67 } 68 } 69 extent_node_init(node, NULL, addr, csize, true); 70 return (node); 71} 72 73/* 74 * base_alloc() guarantees demand-zeroed memory, in order to make multi-page 75 * sparse data structures such as radix tree nodes efficient with respect to 76 * physical memory usage. 77 */ 78void * 79base_alloc(size_t size) 80{ 81 void *ret; 82 size_t csize, usize; 83 extent_node_t *node; 84 extent_node_t key; 85 86 /* 87 * Round size up to nearest multiple of the cacheline size, so that 88 * there is no chance of false cache line sharing. 89 */ 90 csize = CACHELINE_CEILING(size); 91 92 usize = s2u(csize); 93 extent_node_init(&key, NULL, NULL, usize, false); 94 malloc_mutex_lock(&base_mtx); 95 node = extent_tree_szad_nsearch(&base_avail_szad, &key); 96 if (node != NULL) { 97 /* Use existing space. */ 98 extent_tree_szad_remove(&base_avail_szad, node); 99 } else { 100 /* Try to allocate more space. */ 101 node = base_chunk_alloc(csize); 102 } 103 if (node == NULL) { 104 ret = NULL; 105 goto label_return; 106 } 107 108 ret = extent_node_addr_get(node); 109 if (extent_node_size_get(node) > csize) { 110 extent_node_addr_set(node, (void *)((uintptr_t)ret + csize)); 111 extent_node_size_set(node, extent_node_size_get(node) - csize); 112 extent_tree_szad_insert(&base_avail_szad, node); 113 } else 114 base_node_dalloc(node); 115 if (config_stats) { 116 base_allocated += csize; 117 /* 118 * Add one PAGE to base_resident for every page boundary that is 119 * crossed by the new allocation. 120 */ 121 base_resident += PAGE_CEILING((uintptr_t)ret + csize) - 122 PAGE_CEILING((uintptr_t)ret); 123 } 124 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, csize); 125label_return: 126 malloc_mutex_unlock(&base_mtx); 127 return (ret); 128} 129 130void 131base_stats_get(size_t *allocated, size_t *resident, size_t *mapped) 132{ 133 134 malloc_mutex_lock(&base_mtx); 135 *allocated = base_allocated; 136 *resident = base_resident; 137 *mapped = base_mapped; 138 malloc_mutex_unlock(&base_mtx); 139} 140 141bool 142base_boot(void) 143{ 144 145 if (malloc_mutex_init(&base_mtx)) 146 return (true); 147 extent_tree_szad_new(&base_avail_szad); 148 base_nodes = NULL; 149 150 return (false); 151} 152 153void 154base_prefork(void) 155{ 156 157 malloc_mutex_prefork(&base_mtx); 158} 159 160void 161base_postfork_parent(void) 162{ 163 164 malloc_mutex_postfork_parent(&base_mtx); 165} 166 167void 168base_postfork_child(void) 169{ 170 171 malloc_mutex_postfork_child(&base_mtx); 172} 173