1e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define JEMALLOC_CHUNK_C_ 2376b1529a383c39adf4674baf6db83a5e63f97acJason Evans#include "jemalloc/internal/jemalloc_internal.h" 3e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 5e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Data. */ 6e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 7609ae595f0358157b19311b0f9f9591db7cee705Jason Evansconst char *opt_dss = DSS_DEFAULT; 8609ae595f0358157b19311b0f9f9591db7cee705Jason Evanssize_t opt_lg_chunk = LG_CHUNK_DEFAULT; 9e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 103c2343518c2b1fbbd66065c75a3c19f908de1d78Jason Evansmalloc_mutex_t chunks_mtx; 11e476f8a161d445211fd6e54fe370275196e66bcbJason Evanschunk_stats_t stats_chunks; 12e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 137ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans/* 147ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans * Trees of chunks that were previously allocated (trees differ only in node 157ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans * ordering). These are used when allocating chunks, in an attempt to re-use 167ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans * address space. Depending on function, different tree orderings are needed, 177ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans * which is why there are two trees with the same contents. 187ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans */ 19609ae595f0358157b19311b0f9f9591db7cee705Jason Evansstatic extent_tree_t chunks_szad_mmap; 20609ae595f0358157b19311b0f9f9591db7cee705Jason Evansstatic extent_tree_t chunks_ad_mmap; 21609ae595f0358157b19311b0f9f9591db7cee705Jason Evansstatic extent_tree_t chunks_szad_dss; 22609ae595f0358157b19311b0f9f9591db7cee705Jason Evansstatic extent_tree_t chunks_ad_dss; 237ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans 242dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evansrtree_t *chunks_rtree; 252dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans 26e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Various chunk-related settings. */ 27e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t chunksize; 28e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t chunksize_mask; /* (chunksize - 1). */ 29e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t chunk_npages; 307393f44ff025ca67716fc53b68003fd65122fd97Jason Evanssize_t map_bias; 31e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t arena_maxclass; /* Max size class for arenas. */ 32e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 33e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 3412141150fdbda57651a53ae2fe0edaea4891d814Jason Evans/* 3512141150fdbda57651a53ae2fe0edaea4891d814Jason Evans * Function prototypes for static functions that are referenced prior to 3612141150fdbda57651a53ae2fe0edaea4891d814Jason Evans * definition. 3712141150fdbda57651a53ae2fe0edaea4891d814Jason Evans */ 387ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans 3912141150fdbda57651a53ae2fe0edaea4891d814Jason Evansstatic void chunk_dalloc_core(void *chunk, size_t size); 407ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans 417ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans/******************************************************************************/ 427ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans 437ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evansstatic void * 44609ae595f0358157b19311b0f9f9591db7cee705Jason Evanschunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, size_t size, 45609ae595f0358157b19311b0f9f9591db7cee705Jason Evans size_t alignment, bool base, bool *zero) 467ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans{ 477ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans void *ret; 487ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans extent_node_t *node; 497ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans extent_node_t key; 507ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans size_t alloc_size, leadsize, trailsize; 517de92767c20cb72c94609b9c78985526fb84a679Jason Evans bool zeroed; 527ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans 5334a8cf6c4029c09e1db776b7027a9c1b31f9e5b4Jason Evans if (base) { 5434a8cf6c4029c09e1db776b7027a9c1b31f9e5b4Jason Evans /* 5534a8cf6c4029c09e1db776b7027a9c1b31f9e5b4Jason Evans * This function may need to call base_node_{,de}alloc(), but 5634a8cf6c4029c09e1db776b7027a9c1b31f9e5b4Jason Evans * the current chunk allocation request is on behalf of the 5734a8cf6c4029c09e1db776b7027a9c1b31f9e5b4Jason Evans * base allocator. Avoid deadlock (and if that weren't an 5834a8cf6c4029c09e1db776b7027a9c1b31f9e5b4Jason Evans * issue, potential for infinite recursion) by returning NULL. 5934a8cf6c4029c09e1db776b7027a9c1b31f9e5b4Jason Evans */ 6034a8cf6c4029c09e1db776b7027a9c1b31f9e5b4Jason Evans return (NULL); 6134a8cf6c4029c09e1db776b7027a9c1b31f9e5b4Jason Evans } 6234a8cf6c4029c09e1db776b7027a9c1b31f9e5b4Jason Evans 637ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans alloc_size = size + alignment - chunksize; 647ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans /* Beware size_t wrap-around. */ 657ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans if (alloc_size < size) 667ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans return (NULL); 677ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans key.addr = NULL; 687ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans key.size = alloc_size; 697ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans malloc_mutex_lock(&chunks_mtx); 70609ae595f0358157b19311b0f9f9591db7cee705Jason Evans node = extent_tree_szad_nsearch(chunks_szad, &key); 717ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans if (node == NULL) { 727ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans malloc_mutex_unlock(&chunks_mtx); 737ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans return (NULL); 747ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans } 757ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans leadsize = ALIGNMENT_CEILING((uintptr_t)node->addr, alignment) - 767ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans (uintptr_t)node->addr; 77374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans assert(node->size >= leadsize + size); 78374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans trailsize = node->size - leadsize - size; 797ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans ret = (void *)((uintptr_t)node->addr + leadsize); 8014a2c6a698a207ac3f3825443cf3441c8842e990Jason Evans zeroed = node->zeroed; 8114a2c6a698a207ac3f3825443cf3441c8842e990Jason Evans if (zeroed) 8214a2c6a698a207ac3f3825443cf3441c8842e990Jason Evans *zero = true; 837ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans /* Remove node from the tree. */ 84609ae595f0358157b19311b0f9f9591db7cee705Jason Evans extent_tree_szad_remove(chunks_szad, node); 85609ae595f0358157b19311b0f9f9591db7cee705Jason Evans extent_tree_ad_remove(chunks_ad, node); 867ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans if (leadsize != 0) { 877ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans /* Insert the leading space as a smaller chunk. */ 887ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans node->size = leadsize; 89609ae595f0358157b19311b0f9f9591db7cee705Jason Evans extent_tree_szad_insert(chunks_szad, node); 90609ae595f0358157b19311b0f9f9591db7cee705Jason Evans extent_tree_ad_insert(chunks_ad, node); 917ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans node = NULL; 927ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans } 937ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans if (trailsize != 0) { 947ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans /* Insert the trailing space as a smaller chunk. */ 957ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans if (node == NULL) { 967ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans /* 977ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans * An additional node is required, but 987ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans * base_node_alloc() can cause a new base chunk to be 997ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans * allocated. Drop chunks_mtx in order to avoid 1007ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans * deadlock, and if node allocation fails, deallocate 1017ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans * the result before returning an error. 1027ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans */ 1037ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans malloc_mutex_unlock(&chunks_mtx); 1047ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans node = base_node_alloc(); 1057ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans if (node == NULL) { 10612141150fdbda57651a53ae2fe0edaea4891d814Jason Evans chunk_dalloc_core(ret, size); 1077ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans return (NULL); 1087ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans } 1097ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans malloc_mutex_lock(&chunks_mtx); 1107ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans } 1117ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans node->addr = (void *)((uintptr_t)(ret) + size); 1127ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans node->size = trailsize; 113a7a28c334e5526ba716bf6046eab8d60598183ebJason Evans node->zeroed = zeroed; 114609ae595f0358157b19311b0f9f9591db7cee705Jason Evans extent_tree_szad_insert(chunks_szad, node); 115609ae595f0358157b19311b0f9f9591db7cee705Jason Evans extent_tree_ad_insert(chunks_ad, node); 1167ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans node = NULL; 1177ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans } 1187ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans malloc_mutex_unlock(&chunks_mtx); 1197ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans 12014a2c6a698a207ac3f3825443cf3441c8842e990Jason Evans if (node != NULL) 12112141150fdbda57651a53ae2fe0edaea4891d814Jason Evans base_node_dalloc(node); 12214a2c6a698a207ac3f3825443cf3441c8842e990Jason Evans if (*zero) { 12314a2c6a698a207ac3f3825443cf3441c8842e990Jason Evans if (zeroed == false) 12414a2c6a698a207ac3f3825443cf3441c8842e990Jason Evans memset(ret, 0, size); 12514a2c6a698a207ac3f3825443cf3441c8842e990Jason Evans else if (config_debug) { 12614a2c6a698a207ac3f3825443cf3441c8842e990Jason Evans size_t i; 12714a2c6a698a207ac3f3825443cf3441c8842e990Jason Evans size_t *p = (size_t *)(uintptr_t)ret; 12814a2c6a698a207ac3f3825443cf3441c8842e990Jason Evans 129bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans JEMALLOC_VALGRIND_MAKE_MEM_DEFINED(ret, size); 13014a2c6a698a207ac3f3825443cf3441c8842e990Jason Evans for (i = 0; i < size / sizeof(size_t); i++) 13114a2c6a698a207ac3f3825443cf3441c8842e990Jason Evans assert(p[i] == 0); 13214a2c6a698a207ac3f3825443cf3441c8842e990Jason Evans } 13314a2c6a698a207ac3f3825443cf3441c8842e990Jason Evans } 1347ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans return (ret); 1357ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans} 136e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 13741631d00618d7262125e501c91d31b4d70e605faJason Evans/* 13841631d00618d7262125e501c91d31b4d70e605faJason Evans * If the caller specifies (*zero == false), it is still possible to receive 13941631d00618d7262125e501c91d31b4d70e605faJason Evans * zeroed memory, in which case *zero is toggled to true. arena_chunk_alloc() 14041631d00618d7262125e501c91d31b4d70e605faJason Evans * takes advantage of this to avoid demanding zeroed chunks, but taking 14141631d00618d7262125e501c91d31b4d70e605faJason Evans * advantage of them if they are returned. 14241631d00618d7262125e501c91d31b4d70e605faJason Evans */ 14359113bcc94b9fc7549611afb99ca99cad1a7f196aravindstatic void * 14459113bcc94b9fc7549611afb99ca99cad1a7f196aravindchunk_alloc_core(size_t size, size_t alignment, bool base, bool *zero, 145609ae595f0358157b19311b0f9f9591db7cee705Jason Evans dss_prec_t dss_prec) 146e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 147e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 148e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 149e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size != 0); 150e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((size & chunksize_mask) == 0); 151de6fbdb72c6e1401b36f8f2073404645bac6cd2bJason Evans assert(alignment != 0); 152eae269036c9f702d9fa9be497a1a2aa1be13a29eMike Hommey assert((alignment & chunksize_mask) == 0); 153e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 154609ae595f0358157b19311b0f9f9591db7cee705Jason Evans /* "primary" dss. */ 1554d434adb146375ad17f0d5e994ed5728d2942e3fJason Evans if (have_dss && dss_prec == dss_prec_primary) { 15612efefb1953062795f5a971c1a72706787c7895cJason Evans if ((ret = chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, size, 15712efefb1953062795f5a971c1a72706787c7895cJason Evans alignment, base, zero)) != NULL) 15859113bcc94b9fc7549611afb99ca99cad1a7f196aravind return (ret); 15912efefb1953062795f5a971c1a72706787c7895cJason Evans if ((ret = chunk_alloc_dss(size, alignment, zero)) != NULL) 16059113bcc94b9fc7549611afb99ca99cad1a7f196aravind return (ret); 16112efefb1953062795f5a971c1a72706787c7895cJason Evans } 162609ae595f0358157b19311b0f9f9591db7cee705Jason Evans /* mmap. */ 163609ae595f0358157b19311b0f9f9591db7cee705Jason Evans if ((ret = chunk_recycle(&chunks_szad_mmap, &chunks_ad_mmap, size, 164609ae595f0358157b19311b0f9f9591db7cee705Jason Evans alignment, base, zero)) != NULL) 16559113bcc94b9fc7549611afb99ca99cad1a7f196aravind return (ret); 166609ae595f0358157b19311b0f9f9591db7cee705Jason Evans if ((ret = chunk_alloc_mmap(size, alignment, zero)) != NULL) 16759113bcc94b9fc7549611afb99ca99cad1a7f196aravind return (ret); 168609ae595f0358157b19311b0f9f9591db7cee705Jason Evans /* "secondary" dss. */ 1694d434adb146375ad17f0d5e994ed5728d2942e3fJason Evans if (have_dss && dss_prec == dss_prec_secondary) { 17012efefb1953062795f5a971c1a72706787c7895cJason Evans if ((ret = chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, size, 17112efefb1953062795f5a971c1a72706787c7895cJason Evans alignment, base, zero)) != NULL) 17259113bcc94b9fc7549611afb99ca99cad1a7f196aravind return (ret); 17312efefb1953062795f5a971c1a72706787c7895cJason Evans if ((ret = chunk_alloc_dss(size, alignment, zero)) != NULL) 17459113bcc94b9fc7549611afb99ca99cad1a7f196aravind return (ret); 17512efefb1953062795f5a971c1a72706787c7895cJason Evans } 176e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 177e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* All strategies for allocation failed. */ 17859113bcc94b9fc7549611afb99ca99cad1a7f196aravind return (NULL); 17959113bcc94b9fc7549611afb99ca99cad1a7f196aravind} 18059113bcc94b9fc7549611afb99ca99cad1a7f196aravind 18112141150fdbda57651a53ae2fe0edaea4891d814Jason Evansstatic bool 18212141150fdbda57651a53ae2fe0edaea4891d814Jason Evanschunk_register(void *chunk, size_t size, bool base) 18359113bcc94b9fc7549611afb99ca99cad1a7f196aravind{ 18459113bcc94b9fc7549611afb99ca99cad1a7f196aravind 18512141150fdbda57651a53ae2fe0edaea4891d814Jason Evans assert(chunk != NULL); 18612141150fdbda57651a53ae2fe0edaea4891d814Jason Evans assert(CHUNK_ADDR2BASE(chunk) == chunk); 18712141150fdbda57651a53ae2fe0edaea4891d814Jason Evans 18812141150fdbda57651a53ae2fe0edaea4891d814Jason Evans if (config_ivsalloc && base == false) { 18912141150fdbda57651a53ae2fe0edaea4891d814Jason Evans if (rtree_set(chunks_rtree, (uintptr_t)chunk, 1)) 19012141150fdbda57651a53ae2fe0edaea4891d814Jason Evans return (true); 19112141150fdbda57651a53ae2fe0edaea4891d814Jason Evans } 19212141150fdbda57651a53ae2fe0edaea4891d814Jason Evans if (config_stats || config_prof) { 19312141150fdbda57651a53ae2fe0edaea4891d814Jason Evans bool gdump; 19412141150fdbda57651a53ae2fe0edaea4891d814Jason Evans malloc_mutex_lock(&chunks_mtx); 19512141150fdbda57651a53ae2fe0edaea4891d814Jason Evans if (config_stats) 19612141150fdbda57651a53ae2fe0edaea4891d814Jason Evans stats_chunks.nchunks += (size / chunksize); 19712141150fdbda57651a53ae2fe0edaea4891d814Jason Evans stats_chunks.curchunks += (size / chunksize); 19812141150fdbda57651a53ae2fe0edaea4891d814Jason Evans if (stats_chunks.curchunks > stats_chunks.highchunks) { 19912141150fdbda57651a53ae2fe0edaea4891d814Jason Evans stats_chunks.highchunks = 20012141150fdbda57651a53ae2fe0edaea4891d814Jason Evans stats_chunks.curchunks; 20112141150fdbda57651a53ae2fe0edaea4891d814Jason Evans if (config_prof) 20212141150fdbda57651a53ae2fe0edaea4891d814Jason Evans gdump = true; 20312141150fdbda57651a53ae2fe0edaea4891d814Jason Evans } else if (config_prof) 20412141150fdbda57651a53ae2fe0edaea4891d814Jason Evans gdump = false; 20512141150fdbda57651a53ae2fe0edaea4891d814Jason Evans malloc_mutex_unlock(&chunks_mtx); 20612141150fdbda57651a53ae2fe0edaea4891d814Jason Evans if (config_prof && opt_prof && opt_prof_gdump && gdump) 20712141150fdbda57651a53ae2fe0edaea4891d814Jason Evans prof_gdump(); 20812141150fdbda57651a53ae2fe0edaea4891d814Jason Evans } 20912141150fdbda57651a53ae2fe0edaea4891d814Jason Evans if (config_valgrind) 21012141150fdbda57651a53ae2fe0edaea4891d814Jason Evans JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(chunk, size); 21112141150fdbda57651a53ae2fe0edaea4891d814Jason Evans return (false); 21259113bcc94b9fc7549611afb99ca99cad1a7f196aravind} 21359113bcc94b9fc7549611afb99ca99cad1a7f196aravind 21459113bcc94b9fc7549611afb99ca99cad1a7f196aravindvoid * 21512141150fdbda57651a53ae2fe0edaea4891d814Jason Evanschunk_alloc_base(size_t size) 21659113bcc94b9fc7549611afb99ca99cad1a7f196aravind{ 21759113bcc94b9fc7549611afb99ca99cad1a7f196aravind void *ret; 21812141150fdbda57651a53ae2fe0edaea4891d814Jason Evans bool zero; 21959113bcc94b9fc7549611afb99ca99cad1a7f196aravind 22012141150fdbda57651a53ae2fe0edaea4891d814Jason Evans zero = false; 22112141150fdbda57651a53ae2fe0edaea4891d814Jason Evans ret = chunk_alloc_core(size, chunksize, true, &zero, 22212141150fdbda57651a53ae2fe0edaea4891d814Jason Evans chunk_dss_prec_get()); 22312141150fdbda57651a53ae2fe0edaea4891d814Jason Evans if (ret == NULL) 22412141150fdbda57651a53ae2fe0edaea4891d814Jason Evans return (NULL); 22512141150fdbda57651a53ae2fe0edaea4891d814Jason Evans if (chunk_register(ret, size, true)) { 22612141150fdbda57651a53ae2fe0edaea4891d814Jason Evans chunk_dalloc_core(ret, size); 22712141150fdbda57651a53ae2fe0edaea4891d814Jason Evans return (NULL); 22812141150fdbda57651a53ae2fe0edaea4891d814Jason Evans } 22912141150fdbda57651a53ae2fe0edaea4891d814Jason Evans return (ret); 23012141150fdbda57651a53ae2fe0edaea4891d814Jason Evans} 23159113bcc94b9fc7549611afb99ca99cad1a7f196aravind 23212141150fdbda57651a53ae2fe0edaea4891d814Jason Evansvoid * 23312141150fdbda57651a53ae2fe0edaea4891d814Jason Evanschunk_alloc_arena(chunk_alloc_t *chunk_alloc, chunk_dalloc_t *chunk_dalloc, 23412141150fdbda57651a53ae2fe0edaea4891d814Jason Evans unsigned arena_ind, size_t size, size_t alignment, bool *zero) 23512141150fdbda57651a53ae2fe0edaea4891d814Jason Evans{ 23612141150fdbda57651a53ae2fe0edaea4891d814Jason Evans void *ret; 23712141150fdbda57651a53ae2fe0edaea4891d814Jason Evans 23812141150fdbda57651a53ae2fe0edaea4891d814Jason Evans ret = chunk_alloc(size, alignment, zero, arena_ind); 23912141150fdbda57651a53ae2fe0edaea4891d814Jason Evans if (ret != NULL && chunk_register(ret, size, false)) { 24012141150fdbda57651a53ae2fe0edaea4891d814Jason Evans chunk_dalloc(ret, size, arena_ind); 24112141150fdbda57651a53ae2fe0edaea4891d814Jason Evans ret = NULL; 242e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 24312141150fdbda57651a53ae2fe0edaea4891d814Jason Evans 244e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 245e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 246e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 24712141150fdbda57651a53ae2fe0edaea4891d814Jason Evans/* Default arena chunk allocation routine in the absence of user override. */ 24812141150fdbda57651a53ae2fe0edaea4891d814Jason Evansvoid * 24912141150fdbda57651a53ae2fe0edaea4891d814Jason Evanschunk_alloc_default(size_t size, size_t alignment, bool *zero, 25012141150fdbda57651a53ae2fe0edaea4891d814Jason Evans unsigned arena_ind) 25112141150fdbda57651a53ae2fe0edaea4891d814Jason Evans{ 25212141150fdbda57651a53ae2fe0edaea4891d814Jason Evans 25312141150fdbda57651a53ae2fe0edaea4891d814Jason Evans return (chunk_alloc_core(size, alignment, false, zero, 25412141150fdbda57651a53ae2fe0edaea4891d814Jason Evans arenas[arena_ind]->dss_prec)); 25512141150fdbda57651a53ae2fe0edaea4891d814Jason Evans} 25612141150fdbda57651a53ae2fe0edaea4891d814Jason Evans 2577ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evansstatic void 258609ae595f0358157b19311b0f9f9591db7cee705Jason Evanschunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk, 259609ae595f0358157b19311b0f9f9591db7cee705Jason Evans size_t size) 2607ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans{ 2617de92767c20cb72c94609b9c78985526fb84a679Jason Evans bool unzeroed; 2624f929aa94853ecd7da2791f462d1b972ee66db8eJason Evans extent_node_t *xnode, *node, *prev, *xprev, key; 2637ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans 2647de92767c20cb72c94609b9c78985526fb84a679Jason Evans unzeroed = pages_purge(chunk, size); 265bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans JEMALLOC_VALGRIND_MAKE_MEM_NOACCESS(chunk, size); 2667ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans 267374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans /* 268374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans * Allocate a node before acquiring chunks_mtx even though it might not 269374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans * be needed, because base_node_alloc() may cause a new base chunk to 270374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans * be allocated, which could cause deadlock if chunks_mtx were already 271374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans * held. 272374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans */ 273374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans xnode = base_node_alloc(); 2744f929aa94853ecd7da2791f462d1b972ee66db8eJason Evans /* Use xprev to implement conditional deferred deallocation of prev. */ 2754f929aa94853ecd7da2791f462d1b972ee66db8eJason Evans xprev = NULL; 276374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans 2777ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans malloc_mutex_lock(&chunks_mtx); 278374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans key.addr = (void *)((uintptr_t)chunk + size); 279609ae595f0358157b19311b0f9f9591db7cee705Jason Evans node = extent_tree_ad_nsearch(chunks_ad, &key); 280374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans /* Try to coalesce forward. */ 281374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans if (node != NULL && node->addr == key.addr) { 282374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans /* 283374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans * Coalesce chunk with the following address range. This does 284374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans * not change the position within chunks_ad, so only 285374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans * remove/insert from/into chunks_szad. 286374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans */ 287609ae595f0358157b19311b0f9f9591db7cee705Jason Evans extent_tree_szad_remove(chunks_szad, node); 288374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans node->addr = chunk; 289374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans node->size += size; 2907de92767c20cb72c94609b9c78985526fb84a679Jason Evans node->zeroed = (node->zeroed && (unzeroed == false)); 291609ae595f0358157b19311b0f9f9591db7cee705Jason Evans extent_tree_szad_insert(chunks_szad, node); 292374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans } else { 293374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans /* Coalescing forward failed, so insert a new node. */ 294374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans if (xnode == NULL) { 2957ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans /* 296374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans * base_node_alloc() failed, which is an exceedingly 297374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans * unlikely failure. Leak chunk; its pages have 298374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans * already been purged, so this is only a virtual 299374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans * memory leak. 3007ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans */ 301741fbc6ba4499da39dd7d0c067c859fa52f1023fJason Evans goto label_return; 3027ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans } 303374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans node = xnode; 304741fbc6ba4499da39dd7d0c067c859fa52f1023fJason Evans xnode = NULL; /* Prevent deallocation below. */ 305374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans node->addr = chunk; 306374d26a43bcceb12eb56ed7cc47815d7f933901cJason Evans node->size = size; 3077de92767c20cb72c94609b9c78985526fb84a679Jason Evans node->zeroed = (unzeroed == false); 308609ae595f0358157b19311b0f9f9591db7cee705Jason Evans extent_tree_ad_insert(chunks_ad, node); 309609ae595f0358157b19311b0f9f9591db7cee705Jason Evans extent_tree_szad_insert(chunks_szad, node); 3107ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans } 3117ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans 3127ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans /* Try to coalesce backward. */ 313609ae595f0358157b19311b0f9f9591db7cee705Jason Evans prev = extent_tree_ad_prev(chunks_ad, node); 3147ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans if (prev != NULL && (void *)((uintptr_t)prev->addr + prev->size) == 3157ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans chunk) { 3167ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans /* 3177ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans * Coalesce chunk with the previous address range. This does 3187ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans * not change the position within chunks_ad, so only 3197ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans * remove/insert node from/into chunks_szad. 3207ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans */ 321609ae595f0358157b19311b0f9f9591db7cee705Jason Evans extent_tree_szad_remove(chunks_szad, prev); 322609ae595f0358157b19311b0f9f9591db7cee705Jason Evans extent_tree_ad_remove(chunks_ad, prev); 3237ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans 324609ae595f0358157b19311b0f9f9591db7cee705Jason Evans extent_tree_szad_remove(chunks_szad, node); 3257ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans node->addr = prev->addr; 3267ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans node->size += prev->size; 3277de92767c20cb72c94609b9c78985526fb84a679Jason Evans node->zeroed = (node->zeroed && prev->zeroed); 328609ae595f0358157b19311b0f9f9591db7cee705Jason Evans extent_tree_szad_insert(chunks_szad, node); 3297ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans 3304f929aa94853ecd7da2791f462d1b972ee66db8eJason Evans xprev = prev; 3317ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans } 332741fbc6ba4499da39dd7d0c067c859fa52f1023fJason Evans 333741fbc6ba4499da39dd7d0c067c859fa52f1023fJason Evanslabel_return: 3347ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans malloc_mutex_unlock(&chunks_mtx); 3354f929aa94853ecd7da2791f462d1b972ee66db8eJason Evans /* 3364f929aa94853ecd7da2791f462d1b972ee66db8eJason Evans * Deallocate xnode and/or xprev after unlocking chunks_mtx in order to 3374f929aa94853ecd7da2791f462d1b972ee66db8eJason Evans * avoid potential deadlock. 3384f929aa94853ecd7da2791f462d1b972ee66db8eJason Evans */ 3394f929aa94853ecd7da2791f462d1b972ee66db8eJason Evans if (xnode != NULL) 34012141150fdbda57651a53ae2fe0edaea4891d814Jason Evans base_node_dalloc(xnode); 3414f929aa94853ecd7da2791f462d1b972ee66db8eJason Evans if (xprev != NULL) 34212141150fdbda57651a53ae2fe0edaea4891d814Jason Evans base_node_dalloc(xprev); 3437ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans} 3447ca0fdfb85b2a9fc7a112e158892c098e004385bJason Evans 345e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 346609ae595f0358157b19311b0f9f9591db7cee705Jason Evanschunk_unmap(void *chunk, size_t size) 347609ae595f0358157b19311b0f9f9591db7cee705Jason Evans{ 348609ae595f0358157b19311b0f9f9591db7cee705Jason Evans assert(chunk != NULL); 349609ae595f0358157b19311b0f9f9591db7cee705Jason Evans assert(CHUNK_ADDR2BASE(chunk) == chunk); 350609ae595f0358157b19311b0f9f9591db7cee705Jason Evans assert(size != 0); 351609ae595f0358157b19311b0f9f9591db7cee705Jason Evans assert((size & chunksize_mask) == 0); 352609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 3534d434adb146375ad17f0d5e994ed5728d2942e3fJason Evans if (have_dss && chunk_in_dss(chunk)) 354609ae595f0358157b19311b0f9f9591db7cee705Jason Evans chunk_record(&chunks_szad_dss, &chunks_ad_dss, chunk, size); 35512141150fdbda57651a53ae2fe0edaea4891d814Jason Evans else if (chunk_dalloc_mmap(chunk, size)) 356609ae595f0358157b19311b0f9f9591db7cee705Jason Evans chunk_record(&chunks_szad_mmap, &chunks_ad_mmap, chunk, size); 357609ae595f0358157b19311b0f9f9591db7cee705Jason Evans} 358609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 35912141150fdbda57651a53ae2fe0edaea4891d814Jason Evansstatic void 36012141150fdbda57651a53ae2fe0edaea4891d814Jason Evanschunk_dalloc_core(void *chunk, size_t size) 361e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 362e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 363e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(chunk != NULL); 364e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(CHUNK_ADDR2BASE(chunk) == chunk); 365e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size != 0); 366e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((size & chunksize_mask) == 0); 367e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 3687372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_ivsalloc) 369b954bc5d3a65966df0ce7801cd6102542b5e894bJason Evans rtree_set(chunks_rtree, (uintptr_t)chunk, 0); 3707372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats || config_prof) { 3717372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans malloc_mutex_lock(&chunks_mtx); 372609ae595f0358157b19311b0f9f9591db7cee705Jason Evans assert(stats_chunks.curchunks >= (size / chunksize)); 3737372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans stats_chunks.curchunks -= (size / chunksize); 3747372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans malloc_mutex_unlock(&chunks_mtx); 3757372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 376e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 37712141150fdbda57651a53ae2fe0edaea4891d814Jason Evans chunk_unmap(chunk, size); 37812141150fdbda57651a53ae2fe0edaea4891d814Jason Evans} 37912141150fdbda57651a53ae2fe0edaea4891d814Jason Evans 38012141150fdbda57651a53ae2fe0edaea4891d814Jason Evans/* Default arena chunk deallocation routine in the absence of user override. */ 38112141150fdbda57651a53ae2fe0edaea4891d814Jason Evansbool 38212141150fdbda57651a53ae2fe0edaea4891d814Jason Evanschunk_dalloc_default(void *chunk, size_t size, unsigned arena_ind) 38312141150fdbda57651a53ae2fe0edaea4891d814Jason Evans{ 38412141150fdbda57651a53ae2fe0edaea4891d814Jason Evans 38512141150fdbda57651a53ae2fe0edaea4891d814Jason Evans chunk_dalloc_core(chunk, size); 38612141150fdbda57651a53ae2fe0edaea4891d814Jason Evans return (false); 387e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 388e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 389e476f8a161d445211fd6e54fe370275196e66bcbJason Evansbool 390a8f8d7540d66ddee7337db80c92890916e1063caJason Evanschunk_boot(void) 391e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 392e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 393e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Set variables according to the value of opt_lg_chunk. */ 3942dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans chunksize = (ZU(1) << opt_lg_chunk); 395ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans assert(chunksize >= PAGE); 396e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunksize_mask = chunksize - 1; 397ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans chunk_npages = (chunksize >> LG_PAGE); 398e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 3997372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats || config_prof) { 4007372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (malloc_mutex_init(&chunks_mtx)) 4017372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans return (true); 4027372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(&stats_chunks, 0, sizeof(chunk_stats_t)); 4037372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 4044d434adb146375ad17f0d5e994ed5728d2942e3fJason Evans if (have_dss && chunk_dss_boot()) 405e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 406609ae595f0358157b19311b0f9f9591db7cee705Jason Evans extent_tree_szad_new(&chunks_szad_mmap); 407609ae595f0358157b19311b0f9f9591db7cee705Jason Evans extent_tree_ad_new(&chunks_ad_mmap); 408609ae595f0358157b19311b0f9f9591db7cee705Jason Evans extent_tree_szad_new(&chunks_szad_dss); 409609ae595f0358157b19311b0f9f9591db7cee705Jason Evans extent_tree_ad_new(&chunks_ad_dss); 4107372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_ivsalloc) { 4117372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans chunks_rtree = rtree_new((ZU(1) << (LG_SIZEOF_PTR+3)) - 412b980cc774a9ccb208a82f4e9ccdcc695d06a960aJason Evans opt_lg_chunk, base_alloc, NULL); 4137372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (chunks_rtree == NULL) 4147372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans return (true); 4157372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 416e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 417e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 418e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 41920f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evans 42020f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evansvoid 42120f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evanschunk_prefork(void) 42220f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evans{ 42320f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evans 424f1c3da8b02489b7728d4275ac011336299eace62Jason Evans malloc_mutex_prefork(&chunks_mtx); 42520f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evans if (config_ivsalloc) 42620f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evans rtree_prefork(chunks_rtree); 42720f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evans chunk_dss_prefork(); 42820f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evans} 42920f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evans 43020f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evansvoid 43120f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evanschunk_postfork_parent(void) 43220f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evans{ 43320f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evans 43420f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evans chunk_dss_postfork_parent(); 43520f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evans if (config_ivsalloc) 43620f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evans rtree_postfork_parent(chunks_rtree); 43720f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evans malloc_mutex_postfork_parent(&chunks_mtx); 43820f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evans} 43920f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evans 44020f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evansvoid 44120f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evanschunk_postfork_child(void) 44220f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evans{ 44320f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evans 44420f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evans chunk_dss_postfork_child(); 44520f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evans if (config_ivsalloc) 44620f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evans rtree_postfork_child(chunks_rtree); 44720f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evans malloc_mutex_postfork_child(&chunks_mtx); 44820f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evans} 449