14201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans#define JEMALLOC_CHUNK_DSS_C_ 2376b1529a383c39adf4674baf6db83a5e63f97acJason Evans#include "jemalloc/internal/jemalloc_internal.h" 34201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans/******************************************************************************/ 44201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans/* Data. */ 54201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans 6609ae595f0358157b19311b0f9f9591db7cee705Jason Evansconst char *dss_prec_names[] = { 7609ae595f0358157b19311b0f9f9591db7cee705Jason Evans "disabled", 8609ae595f0358157b19311b0f9f9591db7cee705Jason Evans "primary", 9609ae595f0358157b19311b0f9f9591db7cee705Jason Evans "secondary", 10609ae595f0358157b19311b0f9f9591db7cee705Jason Evans "N/A" 11609ae595f0358157b19311b0f9f9591db7cee705Jason Evans}; 12609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 13609ae595f0358157b19311b0f9f9591db7cee705Jason Evans/* Current dss precedence default, used when creating new arenas. */ 14609ae595f0358157b19311b0f9f9591db7cee705Jason Evansstatic dss_prec_t dss_prec_default = DSS_PREC_DEFAULT; 15609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 164e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans/* 174e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans * Protects sbrk() calls. This avoids malloc races among threads, though it 184e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans * does not protect against races with threads that call sbrk() directly. 194e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans */ 204e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansstatic malloc_mutex_t dss_mtx; 214201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans 224201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans/* Base address of the DSS. */ 234e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansstatic void *dss_base; 244201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans/* Current end of the DSS, or ((void *)-1) if the DSS is exhausted. */ 254e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansstatic void *dss_prev; 264201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans/* Current upper limit on DSS addresses. */ 274e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansstatic void *dss_max; 284201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans 294201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans/******************************************************************************/ 304201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans 314201af05425b69ee37ffca437aca0cdd604d1e51Jason Evansstatic void * 3266688535969c6dcb234448e590f27df38b4eebdfJason Evanschunk_dss_sbrk(intptr_t increment) 334201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans{ 344201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans 354d434adb146375ad17f0d5e994ed5728d2942e3fJason Evans#ifdef JEMALLOC_DSS 3666688535969c6dcb234448e590f27df38b4eebdfJason Evans return (sbrk(increment)); 3766688535969c6dcb234448e590f27df38b4eebdfJason Evans#else 387ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans not_implemented(); 397ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans return (NULL); 407ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans#endif 4166688535969c6dcb234448e590f27df38b4eebdfJason Evans} 424201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans 43609ae595f0358157b19311b0f9f9591db7cee705Jason Evansdss_prec_t 44609ae595f0358157b19311b0f9f9591db7cee705Jason Evanschunk_dss_prec_get(void) 45609ae595f0358157b19311b0f9f9591db7cee705Jason Evans{ 46609ae595f0358157b19311b0f9f9591db7cee705Jason Evans dss_prec_t ret; 47609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 48551ebc43647521bdd0bc78558b106762b3388928Jason Evans if (!have_dss) 49609ae595f0358157b19311b0f9f9591db7cee705Jason Evans return (dss_prec_disabled); 50609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_lock(&dss_mtx); 51609ae595f0358157b19311b0f9f9591db7cee705Jason Evans ret = dss_prec_default; 52609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_unlock(&dss_mtx); 53609ae595f0358157b19311b0f9f9591db7cee705Jason Evans return (ret); 54609ae595f0358157b19311b0f9f9591db7cee705Jason Evans} 55609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 56609ae595f0358157b19311b0f9f9591db7cee705Jason Evansbool 57609ae595f0358157b19311b0f9f9591db7cee705Jason Evanschunk_dss_prec_set(dss_prec_t dss_prec) 58609ae595f0358157b19311b0f9f9591db7cee705Jason Evans{ 59609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 60551ebc43647521bdd0bc78558b106762b3388928Jason Evans if (!have_dss) 614d434adb146375ad17f0d5e994ed5728d2942e3fJason Evans return (dss_prec != dss_prec_disabled); 62609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_lock(&dss_mtx); 63609ae595f0358157b19311b0f9f9591db7cee705Jason Evans dss_prec_default = dss_prec; 64609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_unlock(&dss_mtx); 65609ae595f0358157b19311b0f9f9591db7cee705Jason Evans return (false); 66609ae595f0358157b19311b0f9f9591db7cee705Jason Evans} 67609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 684201af05425b69ee37ffca437aca0cdd604d1e51Jason Evansvoid * 69cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evanschunk_alloc_dss(arena_t *arena, void *new_addr, size_t size, size_t alignment, 708fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans bool *zero, bool *commit) 714201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans{ 724d434adb146375ad17f0d5e994ed5728d2942e3fJason Evans cassert(have_dss); 73eae269036c9f702d9fa9be497a1a2aa1be13a29eMike Hommey assert(size > 0 && (size & chunksize_mask) == 0); 74eae269036c9f702d9fa9be497a1a2aa1be13a29eMike Hommey assert(alignment > 0 && (alignment & chunksize_mask) == 0); 757372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans 764201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans /* 774201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans * sbrk() uses a signed increment argument, so take care not to 784201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans * interpret a huge allocation request as a negative increment. 794201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans */ 804201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans if ((intptr_t)size < 0) 814201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans return (NULL); 824201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans 834201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans malloc_mutex_lock(&dss_mtx); 844201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans if (dss_prev != (void *)-1) { 854201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans 864201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans /* 874201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans * The loop is necessary to recover from races with other 884201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans * threads that are using the DSS for something other than 894201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans * malloc. 904201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans */ 914201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans do { 9278ae1ac486ffd7953536786c9a5f9dc2bda78858Dmitry-Me void *ret, *cpad, *dss_next; 9378ae1ac486ffd7953536786c9a5f9dc2bda78858Dmitry-Me size_t gap_size, cpad_size; 9478ae1ac486ffd7953536786c9a5f9dc2bda78858Dmitry-Me intptr_t incr; 95879e76a9e57e725e927e77900940967d301a4958Daniel Micay /* Avoid an unnecessary system call. */ 96879e76a9e57e725e927e77900940967d301a4958Daniel Micay if (new_addr != NULL && dss_max != new_addr) 97879e76a9e57e725e927e77900940967d301a4958Daniel Micay break; 98879e76a9e57e725e927e77900940967d301a4958Daniel Micay 994201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans /* Get the current end of the DSS. */ 10066688535969c6dcb234448e590f27df38b4eebdfJason Evans dss_max = chunk_dss_sbrk(0); 101879e76a9e57e725e927e77900940967d301a4958Daniel Micay 102879e76a9e57e725e927e77900940967d301a4958Daniel Micay /* Make sure the earlier condition still holds. */ 103879e76a9e57e725e927e77900940967d301a4958Daniel Micay if (new_addr != NULL && dss_max != new_addr) 104879e76a9e57e725e927e77900940967d301a4958Daniel Micay break; 105879e76a9e57e725e927e77900940967d301a4958Daniel Micay 1064201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans /* 1074201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans * Calculate how much padding is necessary to 1084201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans * chunk-align the end of the DSS. 1094201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans */ 110eae269036c9f702d9fa9be497a1a2aa1be13a29eMike Hommey gap_size = (chunksize - CHUNK_ADDR2OFFSET(dss_max)) & 111eae269036c9f702d9fa9be497a1a2aa1be13a29eMike Hommey chunksize_mask; 112eae269036c9f702d9fa9be497a1a2aa1be13a29eMike Hommey /* 113eae269036c9f702d9fa9be497a1a2aa1be13a29eMike Hommey * Compute how much chunk-aligned pad space (if any) is 114eae269036c9f702d9fa9be497a1a2aa1be13a29eMike Hommey * necessary to satisfy alignment. This space can be 115eae269036c9f702d9fa9be497a1a2aa1be13a29eMike Hommey * recycled for later use. 116eae269036c9f702d9fa9be497a1a2aa1be13a29eMike Hommey */ 117eae269036c9f702d9fa9be497a1a2aa1be13a29eMike Hommey cpad = (void *)((uintptr_t)dss_max + gap_size); 1185ff709c264e52651de25b788692c62ff1f6f389cJason Evans ret = (void *)ALIGNMENT_CEILING((uintptr_t)dss_max, 1195ff709c264e52651de25b788692c62ff1f6f389cJason Evans alignment); 120eae269036c9f702d9fa9be497a1a2aa1be13a29eMike Hommey cpad_size = (uintptr_t)ret - (uintptr_t)cpad; 121eae269036c9f702d9fa9be497a1a2aa1be13a29eMike Hommey dss_next = (void *)((uintptr_t)ret + size); 122eae269036c9f702d9fa9be497a1a2aa1be13a29eMike Hommey if ((uintptr_t)ret < (uintptr_t)dss_max || 123eae269036c9f702d9fa9be497a1a2aa1be13a29eMike Hommey (uintptr_t)dss_next < (uintptr_t)dss_max) { 124eae269036c9f702d9fa9be497a1a2aa1be13a29eMike Hommey /* Wrap-around. */ 125eae269036c9f702d9fa9be497a1a2aa1be13a29eMike Hommey malloc_mutex_unlock(&dss_mtx); 126eae269036c9f702d9fa9be497a1a2aa1be13a29eMike Hommey return (NULL); 1274201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans } 128eae269036c9f702d9fa9be497a1a2aa1be13a29eMike Hommey incr = gap_size + cpad_size + size; 12966688535969c6dcb234448e590f27df38b4eebdfJason Evans dss_prev = chunk_dss_sbrk(incr); 1304201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans if (dss_prev == dss_max) { 1314201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans /* Success. */ 132eae269036c9f702d9fa9be497a1a2aa1be13a29eMike Hommey dss_max = dss_next; 1334201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans malloc_mutex_unlock(&dss_mtx); 134ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans if (cpad_size != 0) { 135b49a334a645b854dbb1649f15c38d646fee66738Jason Evans chunk_hooks_t chunk_hooks = 136b49a334a645b854dbb1649f15c38d646fee66738Jason Evans CHUNK_HOOKS_INITIALIZER; 137b49a334a645b854dbb1649f15c38d646fee66738Jason Evans chunk_dalloc_wrapper(arena, 1388fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans &chunk_hooks, cpad, cpad_size, 1391ae9287a1aec534fa0a805a717f1c4e058ae8433Jason Evans false, true); 140ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans } 1417ad54c1c30e0805e0758690115875f982de46cf2Jason Evans if (*zero) { 142bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED( 143bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans ret, size); 1447ad54c1c30e0805e0758690115875f982de46cf2Jason Evans memset(ret, 0, size); 1457ad54c1c30e0805e0758690115875f982de46cf2Jason Evans } 14603bf5b67be92db3a49f81816dccb5c18c0f2a0c0Jason Evans if (!*commit) 14703bf5b67be92db3a49f81816dccb5c18c0f2a0c0Jason Evans *commit = pages_decommit(ret, size); 1484201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans return (ret); 1494201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans } 1504201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans } while (dss_prev != (void *)-1); 1514201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans } 1524201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans malloc_mutex_unlock(&dss_mtx); 1534201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans 1544201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans return (NULL); 1554201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans} 1564201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans 1574201af05425b69ee37ffca437aca0cdd604d1e51Jason Evansbool 158cfdc8cfbd626e83d38417bd8c73ac018b611e390Jason Evanschunk_in_dss(void *chunk) 159cfdc8cfbd626e83d38417bd8c73ac018b611e390Jason Evans{ 160cfdc8cfbd626e83d38417bd8c73ac018b611e390Jason Evans bool ret; 161cfdc8cfbd626e83d38417bd8c73ac018b611e390Jason Evans 1624d434adb146375ad17f0d5e994ed5728d2942e3fJason Evans cassert(have_dss); 1637372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans 164cfdc8cfbd626e83d38417bd8c73ac018b611e390Jason Evans malloc_mutex_lock(&dss_mtx); 165cfdc8cfbd626e83d38417bd8c73ac018b611e390Jason Evans if ((uintptr_t)chunk >= (uintptr_t)dss_base 166cfdc8cfbd626e83d38417bd8c73ac018b611e390Jason Evans && (uintptr_t)chunk < (uintptr_t)dss_max) 167cfdc8cfbd626e83d38417bd8c73ac018b611e390Jason Evans ret = true; 168cfdc8cfbd626e83d38417bd8c73ac018b611e390Jason Evans else 169cfdc8cfbd626e83d38417bd8c73ac018b611e390Jason Evans ret = false; 170cfdc8cfbd626e83d38417bd8c73ac018b611e390Jason Evans malloc_mutex_unlock(&dss_mtx); 171cfdc8cfbd626e83d38417bd8c73ac018b611e390Jason Evans 172cfdc8cfbd626e83d38417bd8c73ac018b611e390Jason Evans return (ret); 173cfdc8cfbd626e83d38417bd8c73ac018b611e390Jason Evans} 174cfdc8cfbd626e83d38417bd8c73ac018b611e390Jason Evans 175cfdc8cfbd626e83d38417bd8c73ac018b611e390Jason Evansbool 1764201af05425b69ee37ffca437aca0cdd604d1e51Jason Evanschunk_dss_boot(void) 1774201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans{ 1784201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans 1794d434adb146375ad17f0d5e994ed5728d2942e3fJason Evans cassert(have_dss); 1807372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans 1814201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans if (malloc_mutex_init(&dss_mtx)) 1824201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans return (true); 18366688535969c6dcb234448e590f27df38b4eebdfJason Evans dss_base = chunk_dss_sbrk(0); 1844201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans dss_prev = dss_base; 1854201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans dss_max = dss_base; 1864201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans 1874201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans return (false); 1884201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans} 1894201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans 1904e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid 1914e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evanschunk_dss_prefork(void) 1924e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans{ 1934e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 1944d434adb146375ad17f0d5e994ed5728d2942e3fJason Evans if (have_dss) 1954e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_prefork(&dss_mtx); 1964e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans} 1974e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 1984e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid 1994e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evanschunk_dss_postfork_parent(void) 2004e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans{ 2014e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 2024d434adb146375ad17f0d5e994ed5728d2942e3fJason Evans if (have_dss) 2034e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_postfork_parent(&dss_mtx); 2044e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans} 2054e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 2064e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid 2074e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evanschunk_dss_postfork_child(void) 2084e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans{ 2094e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 2104d434adb146375ad17f0d5e994ed5728d2942e3fJason Evans if (have_dss) 2114e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_postfork_child(&dss_mtx); 2124e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans} 2134e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 2144201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans/******************************************************************************/ 215