1368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross/* 2368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross * Copyright (C) 2016 The Android Open Source Project 3368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross * 4368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross * Licensed under the Apache License, Version 2.0 (the "License"); 5368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross * you may not use this file except in compliance with the License. 6368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross * You may obtain a copy of the License at 7368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross * 8368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross * http://www.apache.org/licenses/LICENSE-2.0 9368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross * 10368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross * Unless required by applicable law or agreed to in writing, software 11368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross * distributed under the License is distributed on an "AS IS" BASIS, 12368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross * See the License for the specific language governing permissions and 14368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross * limitations under the License. 15368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross */ 16368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 17368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Crossstatic pthread_mutex_t malloc_disabled_lock = PTHREAD_MUTEX_INITIALIZER; 18368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Crossstatic bool malloc_disabled_tcache; 19368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 20368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Crossstatic void je_iterate_chunk(arena_chunk_t *chunk, 21368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross void (*callback)(uintptr_t ptr, size_t size, void* arg), void* arg); 22368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Crossstatic void je_iterate_small(arena_run_t *run, 23368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross void (*callback)(uintptr_t ptr, size_t size, void* arg), void* arg); 24368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 25368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross/* je_iterate calls callback for each allocation found in the memory region 26368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross * between [base, base+size). base will be rounded down to by the jemalloc 27368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross * chunk size, and base+size will be rounded up to the chunk size. If no memory 28368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross * managed by jemalloc is found in the requested region, je_iterate returns -1 29368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross * and sets errno to EINVAL. 30368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross * 31368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross * je_iterate must be called when no allocations are in progress, either 32368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross * when single-threaded (for example just after a fork), or between 33368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross * jemalloc_prefork() and jemalloc_postfork_parent(). The callback must 34368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross * not attempt to allocate with jemalloc. 35368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross */ 36368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Crossint je_iterate(uintptr_t base, size_t size, 37368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross void (*callback)(uintptr_t ptr, size_t size, void* arg), void* arg) { 38368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 39368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross int error = EINVAL; 40368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross uintptr_t ptr = (uintptr_t)CHUNK_ADDR2BASE(base); 41368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross uintptr_t end = CHUNK_CEILING(base + size); 42368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 43368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross while (ptr < end) { 441e14731d7182460d082d128660d419b26b9c6c39Colin Cross assert(ptr == (uintptr_t)CHUNK_ADDR2BASE(ptr)); 45368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross extent_node_t *node; 46368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 47368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross node = chunk_lookup((void *)ptr, false); 48368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross if (node == NULL) { 49368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross ptr += chunksize; 50368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross continue; 51368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross } 52368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 53368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross assert(extent_node_achunk_get(node) || 54368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross (uintptr_t)extent_node_addr_get(node) == ptr); 55368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 56368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross error = 0; 57368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross if (extent_node_achunk_get(node)) { 58368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross /* Chunk */ 59368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross arena_chunk_t *chunk = (arena_chunk_t *)ptr; 60368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross ptr += chunksize; 61368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 62368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross if (&chunk->node != node) { 63368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross /* Empty retained chunk */ 64368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross continue; 65368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross } 66368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 67368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross je_iterate_chunk(chunk, callback, arg); 68368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross } else if ((uintptr_t)extent_node_addr_get(node) == ptr) { 69368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross /* Huge allocation */ 70368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross callback(ptr, extent_node_size_get(node), arg); 711e14731d7182460d082d128660d419b26b9c6c39Colin Cross ptr = CHUNK_CEILING(ptr + extent_node_size_get(node)); 72368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross } 73368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross } 74368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 75368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross if (error) { 76368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross set_errno(error); 77368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross return -1; 78368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross } 79368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 80368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross return 0; 81368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross} 82368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 83368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross/* Iterate over a valid jemalloc chunk, calling callback for each large 84368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross * allocation run, and calling je_iterate_small for each small allocation run */ 85368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Crossstatic void je_iterate_chunk(arena_chunk_t *chunk, 86368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross void (*callback)(uintptr_t ptr, size_t size, void* arg), void* arg) { 87368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross size_t pageind; 88368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 89368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross pageind = map_bias; 90368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 91368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross while (pageind < chunk_npages) { 92368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross size_t mapbits; 93368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross size_t size; 94368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 95368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross mapbits = arena_mapbits_get(chunk, pageind); 96368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross if (!arena_mapbits_allocated_get(chunk, pageind)) { 97368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross /* Unallocated run */ 98368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross size = arena_mapbits_unallocated_size_get(chunk, pageind); 99368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross } else if (arena_mapbits_large_get(chunk, pageind)) { 100368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross /* Large allocation run */ 101368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross void *rpages; 102368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 103368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross size = arena_mapbits_large_size_get(chunk, pageind); 104368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross rpages = arena_miscelm_to_rpages(arena_miscelm_get(chunk, pageind)); 105368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross callback((uintptr_t)rpages, size, arg); 106368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross } else { 107368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross /* Run of small allocations */ 108368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross szind_t binind; 109368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross arena_run_t *run; 110368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 111368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross assert(arena_mapbits_small_runind_get(chunk, pageind) == pageind); 112368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross binind = arena_mapbits_binind_get(chunk, pageind); 113368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross run = &arena_miscelm_get(chunk, pageind)->run; 114368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross assert(run->binind == binind); 115368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross size = arena_bin_info[binind].run_size; 116368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 117368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross je_iterate_small(run, callback, arg); 118368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross } 119368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross assert(size == PAGE_CEILING(size)); 120368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross assert(size > 0); 121368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross pageind += size >> LG_PAGE; 122368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross } 123368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 124368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross} 125368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 126368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross/* Iterate over a valid jemalloc small allocation run, calling callback for each 127368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross * active allocation. */ 128368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Crossstatic void je_iterate_small(arena_run_t *run, 129368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross void (*callback)(uintptr_t ptr, size_t size, void* arg), void* arg) { 130368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross szind_t binind; 131368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross const arena_bin_info_t *bin_info; 132368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross uint32_t regind; 133368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross uintptr_t ptr; 134368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross void *rpages; 135368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 136368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross binind = run->binind; 137368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross bin_info = &arena_bin_info[binind]; 138368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross rpages = arena_miscelm_to_rpages(arena_run_to_miscelm(run)); 139368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross ptr = (uintptr_t)rpages + bin_info->reg0_offset; 140368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 141368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross for (regind = 0; regind < bin_info->nregs; regind++) { 142368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross if (bitmap_get(run->bitmap, &bin_info->bitmap_info, regind)) { 143368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross callback(ptr, bin_info->reg_size, arg); 144368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross } 145368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross ptr += bin_info->reg_interval; 146368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross } 147368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross} 148368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 149368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Crossstatic void je_malloc_disable_prefork() { 150368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross pthread_mutex_lock(&malloc_disabled_lock); 151368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross} 152368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 153368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Crossstatic void je_malloc_disable_postfork_parent() { 154368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross pthread_mutex_unlock(&malloc_disabled_lock); 155368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross} 156368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 157368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Crossstatic void je_malloc_disable_postfork_child() { 158368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross pthread_mutex_init(&malloc_disabled_lock, NULL); 159368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross} 160368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 161368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Crossvoid je_malloc_disable_init() { 162368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross if (pthread_atfork(je_malloc_disable_prefork, 163368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross je_malloc_disable_postfork_parent, je_malloc_disable_postfork_child) != 0) { 164368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross malloc_write("<jemalloc>: Error in pthread_atfork()\n"); 165368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross if (opt_abort) 166368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross abort(); 167368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross } 168368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross} 169368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 170368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Crossvoid je_malloc_disable() { 171368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross static pthread_once_t once_control = PTHREAD_ONCE_INIT; 172368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross pthread_once(&once_control, je_malloc_disable_init); 173368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 174368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross pthread_mutex_lock(&malloc_disabled_lock); 175368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross bool new_tcache = false; 176368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross size_t old_len = sizeof(malloc_disabled_tcache); 177368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross je_mallctl("thread.tcache.enabled", 178368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross &malloc_disabled_tcache, &old_len, 179368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross &new_tcache, sizeof(new_tcache)); 180368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross jemalloc_prefork(); 181368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross} 182368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross 183368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Crossvoid je_malloc_enable() { 184368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross jemalloc_postfork_parent(); 185368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross if (malloc_disabled_tcache) { 186368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross je_mallctl("thread.tcache.enabled", NULL, NULL, 187368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross &malloc_disabled_tcache, sizeof(malloc_disabled_tcache)); 188368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross } 189368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross pthread_mutex_unlock(&malloc_disabled_lock); 190368f61eb45edd0ed92db68ff0c8c3c0d998ab010Colin Cross} 191