1bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans/******************************************************************************/ 2bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#ifdef JEMALLOC_H_TYPES 3bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans 4bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#ifdef JEMALLOC_VALGRIND 5bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#include <valgrind/valgrind.h> 6bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans 7bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans/* 8bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans * The size that is reported to Valgrind must be consistent through a chain of 9bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans * malloc..realloc..realloc calls. Request size isn't recorded anywhere in 10bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans * jemalloc, so it is critical that all callers of these macros provide usize 11bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans * rather than request size. As a result, buffer overflow detection is 12bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans * technically weakened for the standard API, though it is generally accepted 13bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans * practice to consider any extra bytes reported by malloc_usable_size() as 14bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans * usable space. 15bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans */ 16bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#define JEMALLOC_VALGRIND_MAKE_MEM_NOACCESS(ptr, usize) do { \ 1783e5767ee9a8c68150cca06ae0d27a13ba4fcaf8Christopher Ferris if (unlikely(in_valgrind)) \ 18bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans valgrind_make_mem_noaccess(ptr, usize); \ 19bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans} while (0) 20bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#define JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ptr, usize) do { \ 2183e5767ee9a8c68150cca06ae0d27a13ba4fcaf8Christopher Ferris if (unlikely(in_valgrind)) \ 22bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans valgrind_make_mem_undefined(ptr, usize); \ 23bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans} while (0) 24bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#define JEMALLOC_VALGRIND_MAKE_MEM_DEFINED(ptr, usize) do { \ 2583e5767ee9a8c68150cca06ae0d27a13ba4fcaf8Christopher Ferris if (unlikely(in_valgrind)) \ 26bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans valgrind_make_mem_defined(ptr, usize); \ 27bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans} while (0) 28bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans/* 29bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans * The VALGRIND_MALLOCLIKE_BLOCK() and VALGRIND_RESIZEINPLACE_BLOCK() macro 30bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans * calls must be embedded in macros rather than in functions so that when 31bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans * Valgrind reports errors, there are no extra stack frames in the backtraces. 32bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans */ 33bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#define JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero) do { \ 3483e5767ee9a8c68150cca06ae0d27a13ba4fcaf8Christopher Ferris if (unlikely(in_valgrind && cond)) \ 35bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans VALGRIND_MALLOCLIKE_BLOCK(ptr, usize, p2rz(ptr), zero); \ 36bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans} while (0) 37bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#define JEMALLOC_VALGRIND_REALLOC(maybe_moved, ptr, usize, \ 38bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans ptr_maybe_null, old_ptr, old_usize, old_rzsize, old_ptr_maybe_null, \ 39bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans zero) do { \ 4083e5767ee9a8c68150cca06ae0d27a13ba4fcaf8Christopher Ferris if (unlikely(in_valgrind)) { \ 41bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans size_t rzsize = p2rz(ptr); \ 42bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans \ 43bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans if (!maybe_moved || ptr == old_ptr) { \ 44bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans VALGRIND_RESIZEINPLACE_BLOCK(ptr, old_usize, \ 45bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans usize, rzsize); \ 46bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans if (zero && old_usize < usize) { \ 47bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans valgrind_make_mem_defined( \ 48bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans (void *)((uintptr_t)ptr + \ 49bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans old_usize), usize - old_usize); \ 50bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans } \ 51bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans } else { \ 52bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans if (!old_ptr_maybe_null || old_ptr != NULL) { \ 53bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans valgrind_freelike_block(old_ptr, \ 54bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans old_rzsize); \ 55bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans } \ 56bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans if (!ptr_maybe_null || ptr != NULL) { \ 57bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans size_t copy_size = (old_usize < usize) \ 58bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans ? old_usize : usize; \ 59bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans size_t tail_size = usize - copy_size; \ 60bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans VALGRIND_MALLOCLIKE_BLOCK(ptr, usize, \ 61bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans rzsize, false); \ 62bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans if (copy_size > 0) { \ 63bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans valgrind_make_mem_defined(ptr, \ 64bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans copy_size); \ 65bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans } \ 66bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans if (zero && tail_size > 0) { \ 67bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans valgrind_make_mem_defined( \ 68bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans (void *)((uintptr_t)ptr + \ 69bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans copy_size), tail_size); \ 70bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans } \ 71bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans } \ 72bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans } \ 73bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans } \ 74bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans} while (0) 75bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#define JEMALLOC_VALGRIND_FREE(ptr, rzsize) do { \ 7683e5767ee9a8c68150cca06ae0d27a13ba4fcaf8Christopher Ferris if (unlikely(in_valgrind)) \ 77bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans valgrind_freelike_block(ptr, rzsize); \ 78bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans} while (0) 79bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#else 80bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#define RUNNING_ON_VALGRIND ((unsigned)0) 81bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#define JEMALLOC_VALGRIND_MAKE_MEM_NOACCESS(ptr, usize) do {} while (0) 82bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#define JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ptr, usize) do {} while (0) 83bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#define JEMALLOC_VALGRIND_MAKE_MEM_DEFINED(ptr, usize) do {} while (0) 84bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#define JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero) do {} while (0) 85bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#define JEMALLOC_VALGRIND_REALLOC(maybe_moved, ptr, usize, \ 86bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans ptr_maybe_null, old_ptr, old_usize, old_rzsize, old_ptr_maybe_null, \ 87bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans zero) do {} while (0) 88bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#define JEMALLOC_VALGRIND_FREE(ptr, rzsize) do {} while (0) 89bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#endif 90bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans 91bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#endif /* JEMALLOC_H_TYPES */ 92bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans/******************************************************************************/ 93bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#ifdef JEMALLOC_H_STRUCTS 94bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans 95bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#endif /* JEMALLOC_H_STRUCTS */ 96bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans/******************************************************************************/ 97bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#ifdef JEMALLOC_H_EXTERNS 98bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans 99bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#ifdef JEMALLOC_VALGRIND 100bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evansvoid valgrind_make_mem_noaccess(void *ptr, size_t usize); 101bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evansvoid valgrind_make_mem_undefined(void *ptr, size_t usize); 102bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evansvoid valgrind_make_mem_defined(void *ptr, size_t usize); 103bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evansvoid valgrind_freelike_block(void *ptr, size_t usize); 104bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#endif 105bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans 106bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#endif /* JEMALLOC_H_EXTERNS */ 107bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans/******************************************************************************/ 108bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#ifdef JEMALLOC_H_INLINES 109bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans 110bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans#endif /* JEMALLOC_H_INLINES */ 111bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans/******************************************************************************/ 112bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans 113