prof.h revision 5389146191b279ca3b90028357dd6ad66b283def
146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton/******************************************************************************/ 246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton#ifdef JEMALLOC_H_TYPES 346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Claytontypedef struct prof_bt_s prof_bt_t; 546c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Claytontypedef struct prof_cnt_s prof_cnt_t; 646c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Claytontypedef struct prof_thr_cnt_s prof_thr_cnt_t; 746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Claytontypedef struct prof_ctx_s prof_ctx_t; 846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Claytontypedef struct prof_tdata_s prof_tdata_t; 946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 1046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton/* Option defaults. */ 1146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton#define PROF_PREFIX_DEFAULT "jeprof" 1246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton#define LG_PROF_SAMPLE_DEFAULT 0 1346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton#define LG_PROF_INTERVAL_DEFAULT -1 1446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 1546c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton/* 169ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton * Hard limit on stack backtrace depth. The version of prof_backtrace() that 1746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * is based on __builtin_return_address() necessarily has a hard-coded number 1846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * of backtrace frame handlers, and should be kept in sync with this setting. 1946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton */ 2046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton#define PROF_BT_MAX 128 2149ce8969d3154e1560106cfe530444c09410f217Greg Clayton 2249ce8969d3154e1560106cfe530444c09410f217Greg Clayton/* Maximum number of backtraces to store in each per thread LRU cache. */ 2349ce8969d3154e1560106cfe530444c09410f217Greg Clayton#define PROF_TCMAX 1024 2446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 2546c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton/* Initial hash table size. */ 2649ce8969d3154e1560106cfe530444c09410f217Greg Clayton#define PROF_CKH_MINITEMS 64 2746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 2846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton/* Size of memory buffer to use when writing dump files. */ 2946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton#define PROF_DUMP_BUF_SIZE 65536 3046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 3146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton#endif /* JEMALLOC_H_TYPES */ 3246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton/******************************************************************************/ 3346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton#ifdef JEMALLOC_H_STRUCTS 3446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 354d511cb4a04cce1793d6eab0aad0b138285930c1Greg Claytonstruct prof_bt_s { 369ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton /* Backtrace, stored as len program counters. */ 379ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton void **vec; 389ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton unsigned len; 3946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton}; 4046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 4146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton#ifdef JEMALLOC_PROF_LIBGCC 420e191607adcb0ea8ebd06c278be648a7f5c0097fGreg Clayton/* Data structure passed to libgcc _Unwind_Backtrace() callback functions. */ 4346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Claytontypedef struct { 4446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton prof_bt_t *bt; 450e191607adcb0ea8ebd06c278be648a7f5c0097fGreg Clayton unsigned nignore; 460e191607adcb0ea8ebd06c278be648a7f5c0097fGreg Clayton unsigned max; 4746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton} prof_unwind_data_t; 4846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton#endif 4946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 5046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Claytonstruct prof_cnt_s { 5146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* 5246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * Profiling counters. An allocation/deallocation pair can operate on 5346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * different prof_thr_cnt_t objects that are linked into the same 5446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * prof_ctx_t cnts_ql, so it is possible for the cur* counters to go 5546c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * negative. In principle it is possible for the *bytes counters to 5646c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * overflow/underflow, but a general solution would require something 5746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * like 128-bit counters; this implementation doesn't bother to solve 5846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * that problem. 5946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton */ 6046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton int64_t curobjs; 6146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton int64_t curbytes; 6246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton uint64_t accumobjs; 6346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton uint64_t accumbytes; 6446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton}; 6546c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 6646c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Claytonstruct prof_thr_cnt_s { 6746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* Linkage into prof_ctx_t's cnts_ql. */ 6846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton ql_elm(prof_thr_cnt_t) cnts_link; 6946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 7046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* Linkage into thread's LRU. */ 7146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton ql_elm(prof_thr_cnt_t) lru_link; 7246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 7346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* 7446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * Associated context. If a thread frees an object that it did not 7546c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * allocate, it is possible that the context is not cached in the 7646c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * thread's hash table, in which case it must be able to look up the 7746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * context, insert a new prof_thr_cnt_t into the thread's hash table, 7846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * and link it into the prof_ctx_t's cnts_ql. 7946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton */ 80444fe998bf707bd076a70c3a779db8575533695eGreg Clayton prof_ctx_t *ctx; 81444fe998bf707bd076a70c3a779db8575533695eGreg Clayton 829ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton /* 839ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton * Threads use memory barriers to update the counters. Since there is 849ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton * only ever one writer, the only challenge is for the reader to get a 859ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton * consistent read of the counters. 8646c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * 8746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * The writer uses this series of operations: 8846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * 8946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * 1) Increment epoch to an odd number. 90b170aee2daacc83e3d71c3e3acc9d56c89893a7bGreg Clayton * 2) Update counters. 9146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * 3) Increment epoch to an even number. 9246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * 9346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * The reader must assure 1) that the epoch is even while it reads the 9446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * counters, and 2) that the epoch doesn't change between the time it 9546c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * starts and finishes reading the counters. 9646c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton */ 9746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton unsigned epoch; 9846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 9946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* Profiling counters. */ 10046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton prof_cnt_t cnts; 10146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton}; 10246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 10346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Claytonstruct prof_ctx_s { 10446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* Associated backtrace. */ 10546c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton prof_bt_t *bt; 10646c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 10746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* Protects cnt_merged and cnts_ql. */ 10846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton malloc_mutex_t lock; 1099ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton 1109ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton /* Temporary storage for summation during dump. */ 11146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton prof_cnt_t cnt_summed; 11246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 11346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* When threads exit, they merge their stats into cnt_merged. */ 11446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton prof_cnt_t cnt_merged; 11546c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 11646c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* 11746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * List of profile counters, one for each thread that has allocated in 11846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * this context. 11946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton */ 12046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton ql_head(prof_thr_cnt_t) cnts_ql; 12146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton}; 12246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 12346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Claytonstruct prof_tdata_s { 12446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* 12546c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * Hash of (prof_bt_t *)-->(prof_thr_cnt_t *). Each thread keeps a 12646c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * cache of backtraces, with associated thread-specific prof_thr_cnt_t 12746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * objects. Other threads may read the prof_thr_cnt_t contents, but no 12846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * others will ever write them. 12946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * 1300e191607adcb0ea8ebd06c278be648a7f5c0097fGreg Clayton * Upon thread exit, the thread must merge all the prof_thr_cnt_t 13146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * counter data into the associated prof_ctx_t objects, and unlink/free 13246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * the prof_thr_cnt_t objects. 13346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton */ 13446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton ckh_t bt2cnt; 13546c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 13646c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* LRU for contents of bt2cnt. */ 13746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton ql_head(prof_thr_cnt_t) lru_ql; 13846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 13946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* Backtrace vector, used for calls to prof_backtrace(). */ 14046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton void **vec; 14146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 1429ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton /* Sampling state. */ 1439ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton uint64_t prn_state; 1449ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton uint64_t threshold; 1459ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton uint64_t accum; 1469ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton}; 1479ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton 1489ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton#endif /* JEMALLOC_H_STRUCTS */ 1499ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton/******************************************************************************/ 1509ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton#ifdef JEMALLOC_H_EXTERNS 1519ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton 1529ce953807eb814a93b449dc243de4f7bf32c3115Greg Claytonextern bool opt_prof; 1539ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton/* 1549ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton * Even if opt_prof is true, sampling can be temporarily disabled by setting 1559ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton * opt_prof_active to false. No locking is used when updating opt_prof_active, 1569ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton * so there are no guarantees regarding how long it will take for all threads 1579ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton * to notice state changes. 1589ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton */ 1599ce953807eb814a93b449dc243de4f7bf32c3115Greg Claytonextern bool opt_prof_active; 1609ce953807eb814a93b449dc243de4f7bf32c3115Greg Claytonextern size_t opt_lg_prof_sample; /* Mean bytes between samples. */ 1619ce953807eb814a93b449dc243de4f7bf32c3115Greg Claytonextern ssize_t opt_lg_prof_interval; /* lg(prof_interval). */ 1625f35a4be95aed0e5b2cb36f7d785bcbfc67284aeDaniel Maleaextern bool opt_prof_gdump; /* High-water memory dumping. */ 1639ce953807eb814a93b449dc243de4f7bf32c3115Greg Claytonextern bool opt_prof_leak; /* Dump leak summary at exit. */ 1649ce953807eb814a93b449dc243de4f7bf32c3115Greg Claytonextern bool opt_prof_accum; /* Report cumulative bytes. */ 1659ce953807eb814a93b449dc243de4f7bf32c3115Greg Claytonextern char opt_prof_prefix[PATH_MAX + 1]; 1669ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton 1679ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton/* 1689ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton * Profile dump interval, measured in bytes allocated. Each arena triggers a 1699ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton * profile dump when it reaches this threshold. The effect is that the 1709ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton * interval between profile dumps averages prof_interval, though the actual 1719ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton * interval between dumps will tend to be sporadic, and the interval will be a 1729ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton * maximum of approximately (prof_interval * narenas). 1739ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton */ 1745f35a4be95aed0e5b2cb36f7d785bcbfc67284aeDaniel Maleaextern uint64_t prof_interval; 1759ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton 1769ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton/* 1779ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton * If true, promote small sampled objects to large objects, since small run 1789ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton * headers do not have embedded profile context pointers. 1799ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton */ 1809ce953807eb814a93b449dc243de4f7bf32c3115Greg Claytonextern bool prof_promote; 1815f35a4be95aed0e5b2cb36f7d785bcbfc67284aeDaniel Malea 1829ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton/* Thread-specific backtrace cache, used to reduce bt2ctx contention. */ 1839ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton#ifndef NO_TLS 1849ce953807eb814a93b449dc243de4f7bf32c3115Greg Claytonextern __thread prof_tdata_t *prof_tdata_tls 1859ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton JEMALLOC_ATTR(tls_model("initial-exec")); 1869ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton# define PROF_TCACHE_GET() prof_tdata_tls 1879ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton# define PROF_TCACHE_SET(v) do { \ 1889ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton prof_tdata_tls = (v); \ 1899ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton pthread_setspecific(prof_tdata_tsd, (void *)(v)); \ 1909ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton} while (0) 1919ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton#else 1929ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton# define PROF_TCACHE_GET() \ 1939ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton ((prof_tdata_t *)pthread_getspecific(prof_tdata_tsd)) 1949ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton# define PROF_TCACHE_SET(v) do { \ 1959ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton pthread_setspecific(prof_tdata_tsd, (void *)(v)); \ 1969ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton} while (0) 19746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton#endif 19846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton/* 19946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * Same contents as b2cnt_tls, but initialized such that the TSD destructor is 20046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * called when a thread exits, so that prof_tdata_tls contents can be merged, 20146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * unlinked, and deallocated. 20246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton */ 20346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Claytonextern pthread_key_t prof_tdata_tsd; 2049ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton 2059ce953807eb814a93b449dc243de4f7bf32c3115Greg Claytonvoid bt_init(prof_bt_t *bt, void **vec); 2069ce953807eb814a93b449dc243de4f7bf32c3115Greg Claytonvoid prof_backtrace(prof_bt_t *bt, unsigned nignore); 2079ce953807eb814a93b449dc243de4f7bf32c3115Greg Claytonprof_thr_cnt_t *prof_lookup(prof_bt_t *bt); 2089ce953807eb814a93b449dc243de4f7bf32c3115Greg Claytonvoid prof_idump(void); 2099ce953807eb814a93b449dc243de4f7bf32c3115Greg Claytonbool prof_mdump(const char *filename); 21046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Claytonvoid prof_gdump(void); 21146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Claytonprof_tdata_t *prof_tdata_init(void); 21246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Claytonvoid prof_boot0(void); 21346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Claytonvoid prof_boot1(void); 21446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Claytonbool prof_boot2(void); 21546c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 2168775adaaba9befbbcbc7d6f42ef3a28a917d7c91Greg Clayton#endif /* JEMALLOC_H_EXTERNS */ 2178775adaaba9befbbcbc7d6f42ef3a28a917d7c91Greg Clayton/******************************************************************************/ 2188775adaaba9befbbcbc7d6f42ef3a28a917d7c91Greg Clayton#ifdef JEMALLOC_H_INLINES 2197cba7deac40f09618f8acf8aee08fe80bbb306bfGreg Clayton 2208775adaaba9befbbcbc7d6f42ef3a28a917d7c91Greg Clayton#define PROF_ALLOC_PREP(nignore, size, ret) do { \ 2218775adaaba9befbbcbc7d6f42ef3a28a917d7c91Greg Clayton prof_tdata_t *prof_tdata; \ 2228775adaaba9befbbcbc7d6f42ef3a28a917d7c91Greg Clayton prof_bt_t bt; \ 22346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton \ 22446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton assert(size == s2u(size)); \ 22546c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton \ 22646c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton prof_tdata = PROF_TCACHE_GET(); \ 22746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton if (prof_tdata == NULL) { \ 22846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton prof_tdata = prof_tdata_init(); \ 22946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton if (prof_tdata == NULL) { \ 23046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton ret = NULL; \ 23146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton break; \ 23246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton } \ 23346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton } \ 23446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton \ 23546c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton if (opt_prof_active == false) { \ 2369ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton /* Sampling is currently inactive, so avoid sampling. */\ 237d6d45ceeae51b6dbb9221a3cf82441afa212a7dcJason Molenda ret = (prof_thr_cnt_t *)(uintptr_t)1U; \ 238d6d45ceeae51b6dbb9221a3cf82441afa212a7dcJason Molenda } else if (opt_lg_prof_sample == 0) { \ 2399ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton /* Don't bother with sampling logic, since sampling */\ 2409ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton /* interval is 1. */\ 2419ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton bt_init(&bt, prof_tdata->vec); \ 2429ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton prof_backtrace(&bt, nignore); \ 2439ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton ret = prof_lookup(&bt); \ 2449ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton } else { \ 24546c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton if (prof_tdata->threshold == 0) { \ 24646c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* Initialize. Seed the prng differently for */\ 24746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* each thread. */\ 24846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton prof_tdata->prn_state = \ 24946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton (uint64_t)(uintptr_t)&size; \ 25046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton prof_sample_threshold_update(prof_tdata); \ 25146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton } \ 25246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton \ 25346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* Determine whether to capture a backtrace based on */\ 25446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* whether size is enough for prof_accum to reach */\ 25546c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* prof_tdata->threshold. However, delay updating */\ 25646c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* these variables until prof_{m,re}alloc(), because */\ 25746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* we don't know for sure that the allocation will */\ 25846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* succeed. */\ 25946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* */\ 26046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* Use subtraction rather than addition to avoid */\ 26146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* potential integer overflow. */\ 2625f35a4be95aed0e5b2cb36f7d785bcbfc67284aeDaniel Malea if (size >= prof_tdata->threshold - \ 2639ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton prof_tdata->accum) { \ 2649ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton bt_init(&bt, prof_tdata->vec); \ 2659ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton prof_backtrace(&bt, nignore); \ 2669ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton ret = prof_lookup(&bt); \ 2679ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton } else \ 2689ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton ret = (prof_thr_cnt_t *)(uintptr_t)1U; \ 26946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton } \ 27046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton} while (0) 27146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 27246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton#ifndef JEMALLOC_ENABLE_INLINE 27346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Claytonvoid prof_sample_threshold_update(prof_tdata_t *prof_tdata); 27446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Claytonprof_ctx_t *prof_ctx_get(const void *ptr); 2759ce953807eb814a93b449dc243de4f7bf32c3115Greg Claytonvoid prof_ctx_set(const void *ptr, prof_ctx_t *ctx); 27646c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Claytonbool prof_sample_accum_update(size_t size); 27746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Claytonvoid prof_malloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt); 27846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Claytonvoid prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt, 27946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton size_t old_size, prof_ctx_t *old_ctx); 28046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Claytonvoid prof_free(const void *ptr, size_t size); 2819ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton#endif 2829ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton 2839ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_)) 2849ce953807eb814a93b449dc243de4f7bf32c3115Greg ClaytonJEMALLOC_INLINE void 2859ce953807eb814a93b449dc243de4f7bf32c3115Greg Claytonprof_sample_threshold_update(prof_tdata_t *prof_tdata) 2869ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton{ 2879ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton uint64_t r; 28846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton double u; 28946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 29046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton cassert(config_prof); 29146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 29246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* 29346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * Compute sample threshold as a geometrically distributed random 2949ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton * variable with mean (2^opt_lg_prof_sample). 2959ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton * 2969ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton * __ __ 2979ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton * | log(u) | 1 2989ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton * prof_tdata->threshold = | -------- |, where p = ------------------- 2999ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton * | log(1-p) | opt_lg_prof_sample 3009ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton * 2 3019ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton * 3029ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton * For more information on the math, see: 3039ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton * 3049ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton * Non-Uniform Random Variate Generation 305cd0dd8fa16ba32e4bed9326757b3978ba1155179Jason Molenda * Luc Devroye 306cd0dd8fa16ba32e4bed9326757b3978ba1155179Jason Molenda * Springer-Verlag, New York, 1986 3074d511cb4a04cce1793d6eab0aad0b138285930c1Greg Clayton * pp 500 3084d511cb4a04cce1793d6eab0aad0b138285930c1Greg Clayton * (http://cg.scs.carleton.ca/~luc/rnbookindex.html) 3094d511cb4a04cce1793d6eab0aad0b138285930c1Greg Clayton */ 3104d511cb4a04cce1793d6eab0aad0b138285930c1Greg Clayton prn64(r, 53, prof_tdata->prn_state, 3114d511cb4a04cce1793d6eab0aad0b138285930c1Greg Clayton (uint64_t)6364136223846793005LLU, (uint64_t)1442695040888963407LLU); 3124d511cb4a04cce1793d6eab0aad0b138285930c1Greg Clayton u = (double)r * (1.0/9007199254740992.0L); 3134d511cb4a04cce1793d6eab0aad0b138285930c1Greg Clayton prof_tdata->threshold = (uint64_t)(log(u) / 314cd0dd8fa16ba32e4bed9326757b3978ba1155179Jason Molenda log(1.0 - (1.0 / (double)((uint64_t)1U << opt_lg_prof_sample)))) 3154d511cb4a04cce1793d6eab0aad0b138285930c1Greg Clayton + (uint64_t)1U; 3164d511cb4a04cce1793d6eab0aad0b138285930c1Greg Clayton} 3174d511cb4a04cce1793d6eab0aad0b138285930c1Greg Clayton 3184d511cb4a04cce1793d6eab0aad0b138285930c1Greg ClaytonJEMALLOC_INLINE prof_ctx_t * 3194d511cb4a04cce1793d6eab0aad0b138285930c1Greg Claytonprof_ctx_get(const void *ptr) 3204d511cb4a04cce1793d6eab0aad0b138285930c1Greg Clayton{ 3214d511cb4a04cce1793d6eab0aad0b138285930c1Greg Clayton prof_ctx_t *ret; 3224d511cb4a04cce1793d6eab0aad0b138285930c1Greg Clayton arena_chunk_t *chunk; 3234d511cb4a04cce1793d6eab0aad0b138285930c1Greg Clayton 3244d511cb4a04cce1793d6eab0aad0b138285930c1Greg Clayton cassert(config_prof); 3254d511cb4a04cce1793d6eab0aad0b138285930c1Greg Clayton assert(ptr != NULL); 3264d511cb4a04cce1793d6eab0aad0b138285930c1Greg Clayton 327cd0dd8fa16ba32e4bed9326757b3978ba1155179Jason Molenda chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 328cd0dd8fa16ba32e4bed9326757b3978ba1155179Jason Molenda if (chunk != ptr) { 32946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* Region. */ 33046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton ret = arena_prof_ctx_get(ptr); 33146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton } else 3329ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton ret = huge_prof_ctx_get(ptr); 3339ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton 3349ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton return (ret); 3359ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton} 336f0c0882ff3fd98456c1814b495888092da2f13f6Jason Molenda 3379ce953807eb814a93b449dc243de4f7bf32c3115Greg ClaytonJEMALLOC_INLINE void 3389ce953807eb814a93b449dc243de4f7bf32c3115Greg Claytonprof_ctx_set(const void *ptr, prof_ctx_t *ctx) 33946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton{ 340ae932359b80098532f3c3766fa9e6527352fbb67Greg Clayton arena_chunk_t *chunk; 34146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 34246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton cassert(config_prof); 34346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton assert(ptr != NULL); 34446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 34546c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 34646c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton if (chunk != ptr) { 34746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* Region. */ 34846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton arena_prof_ctx_set(ptr, ctx); 34946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton } else 35046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton huge_prof_ctx_set(ptr, ctx); 35146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton} 35246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 35346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg ClaytonJEMALLOC_INLINE bool 35494a5d0de4433dce556db59758f3d6124eb0e1a2aJim Inghamprof_sample_accum_update(size_t size) 35546c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton{ 35646c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton prof_tdata_t *prof_tdata; 35746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 35846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton cassert(config_prof); 35946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* Sampling logic is unnecessary if the interval is 1. */ 36046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton assert(opt_lg_prof_sample != 0); 36146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 36246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton prof_tdata = PROF_TCACHE_GET(); 36346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton assert(prof_tdata != NULL); 36446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 365ae932359b80098532f3c3766fa9e6527352fbb67Greg Clayton /* Take care to avoid integer overflow. */ 36646c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton if (size >= prof_tdata->threshold - prof_tdata->accum) { 36746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton prof_tdata->accum -= (prof_tdata->threshold - size); 36846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* Compute new sample threshold. */ 36946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton prof_sample_threshold_update(prof_tdata); 37046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton while (prof_tdata->accum >= prof_tdata->threshold) { 37146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton prof_tdata->accum -= prof_tdata->threshold; 37246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton prof_sample_threshold_update(prof_tdata); 37346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton } 37446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton return (false); 37546c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton } else { 37646c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton prof_tdata->accum += size; 37746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton return (true); 37846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton } 37946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton} 38046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 38146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg ClaytonJEMALLOC_INLINE void 38246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Claytonprof_malloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt) 38346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton{ 38446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 38546c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton cassert(config_prof); 38646c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton assert(ptr != NULL); 38746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton assert(size == isalloc(ptr)); 38846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 38946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton if (opt_lg_prof_sample != 0) { 39046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton if (prof_sample_accum_update(size)) { 39146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* 39246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * Don't sample. For malloc()-like allocation, it is 393ba065cab7acc8ef7fbedc27af5d18c3a694a084aJason Molenda * always possible to tell in advance how large an 394ba065cab7acc8ef7fbedc27af5d18c3a694a084aJason Molenda * object's usable size will be, so there should never 395ba065cab7acc8ef7fbedc27af5d18c3a694a084aJason Molenda * be a difference between the size passed to 396ba065cab7acc8ef7fbedc27af5d18c3a694a084aJason Molenda * PROF_ALLOC_PREP() and prof_malloc(). 397ba065cab7acc8ef7fbedc27af5d18c3a694a084aJason Molenda */ 398ba065cab7acc8ef7fbedc27af5d18c3a694a084aJason Molenda assert((uintptr_t)cnt == (uintptr_t)1U); 39946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton } 40046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton } 40146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 40246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton if ((uintptr_t)cnt > (uintptr_t)1U) { 4039ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton prof_ctx_set(ptr, cnt->ctx); 4049ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton 4059ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton cnt->epoch++; 4069ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton /*********/ 4079ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton mb_write(); 4089ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton /*********/ 4099ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton cnt->cnts.curobjs++; 4109ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton cnt->cnts.curbytes += size; 41146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton if (opt_prof_accum) { 41246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton cnt->cnts.accumobjs++; 41346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton cnt->cnts.accumbytes += size; 41446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton } 41546c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /*********/ 41646c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton mb_write(); 41746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /*********/ 41846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton cnt->epoch++; 41946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /*********/ 42046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton mb_write(); 42146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /*********/ 42246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton } else 42346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U); 42446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton} 42546c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 42646c9a355af9b39db78c006b2a5cbf97d3c58d947Greg ClaytonJEMALLOC_INLINE void 42746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Claytonprof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt, 42846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton size_t old_size, prof_ctx_t *old_ctx) 4295f35a4be95aed0e5b2cb36f7d785bcbfc67284aeDaniel Malea{ 43046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton prof_thr_cnt_t *told_cnt; 43146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 43246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton cassert(config_prof); 43346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton assert(ptr != NULL || (uintptr_t)cnt <= (uintptr_t)1U); 43446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 43546c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton if (ptr != NULL) { 43646c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton assert(size == isalloc(ptr)); 43746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton if (opt_lg_prof_sample != 0) { 43846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton if (prof_sample_accum_update(size)) { 43946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* 44046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * Don't sample. The size passed to 44146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * PROF_ALLOC_PREP() was larger than what 44246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * actually got allocated, so a backtrace was 44346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * captured for this allocation, even though 44446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * its actual size was insufficient to cross 44546c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * the sample threshold. 44646c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton */ 44746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton cnt = (prof_thr_cnt_t *)(uintptr_t)1U; 44846c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton } 44946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton } 45046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton } 45146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton 45246c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton if ((uintptr_t)old_ctx > (uintptr_t)1U) { 45346c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton told_cnt = prof_lookup(old_ctx->bt); 45446c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton if (told_cnt == NULL) { 45546c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton /* 45646c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * It's too late to propagate OOM for this realloc(), 45746c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton * so operate directly on old_cnt->ctx->cnt_merged. 4589ce953807eb814a93b449dc243de4f7bf32c3115Greg Clayton */ 45946c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton malloc_mutex_lock(&old_ctx->lock); 46046c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton old_ctx->cnt_merged.curobjs--; 46146c9a355af9b39db78c006b2a5cbf97d3c58d947Greg Clayton old_ctx->cnt_merged.curbytes -= old_size; 46249ce8969d3154e1560106cfe530444c09410f217Greg Clayton malloc_mutex_unlock(&old_ctx->lock); 46349ce8969d3154e1560106cfe530444c09410f217Greg Clayton told_cnt = (prof_thr_cnt_t *)(uintptr_t)1U; 46449ce8969d3154e1560106cfe530444c09410f217Greg Clayton } 46549ce8969d3154e1560106cfe530444c09410f217Greg Clayton } else 46649ce8969d3154e1560106cfe530444c09410f217Greg Clayton told_cnt = (prof_thr_cnt_t *)(uintptr_t)1U; 467 468 if ((uintptr_t)told_cnt > (uintptr_t)1U) 469 told_cnt->epoch++; 470 if ((uintptr_t)cnt > (uintptr_t)1U) { 471 prof_ctx_set(ptr, cnt->ctx); 472 cnt->epoch++; 473 } else 474 prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U); 475 /*********/ 476 mb_write(); 477 /*********/ 478 if ((uintptr_t)told_cnt > (uintptr_t)1U) { 479 told_cnt->cnts.curobjs--; 480 told_cnt->cnts.curbytes -= old_size; 481 } 482 if ((uintptr_t)cnt > (uintptr_t)1U) { 483 cnt->cnts.curobjs++; 484 cnt->cnts.curbytes += size; 485 if (opt_prof_accum) { 486 cnt->cnts.accumobjs++; 487 cnt->cnts.accumbytes += size; 488 } 489 } 490 /*********/ 491 mb_write(); 492 /*********/ 493 if ((uintptr_t)told_cnt > (uintptr_t)1U) 494 told_cnt->epoch++; 495 if ((uintptr_t)cnt > (uintptr_t)1U) 496 cnt->epoch++; 497 /*********/ 498 mb_write(); /* Not strictly necessary. */ 499} 500 501JEMALLOC_INLINE void 502prof_free(const void *ptr, size_t size) 503{ 504 prof_ctx_t *ctx = prof_ctx_get(ptr); 505 506 cassert(config_prof); 507 508 if ((uintptr_t)ctx > (uintptr_t)1) { 509 assert(size == isalloc(ptr)); 510 prof_thr_cnt_t *tcnt = prof_lookup(ctx->bt); 511 512 if (tcnt != NULL) { 513 tcnt->epoch++; 514 /*********/ 515 mb_write(); 516 /*********/ 517 tcnt->cnts.curobjs--; 518 tcnt->cnts.curbytes -= size; 519 /*********/ 520 mb_write(); 521 /*********/ 522 tcnt->epoch++; 523 /*********/ 524 mb_write(); 525 /*********/ 526 } else { 527 /* 528 * OOM during free() cannot be propagated, so operate 529 * directly on cnt->ctx->cnt_merged. 530 */ 531 malloc_mutex_lock(&ctx->lock); 532 ctx->cnt_merged.curobjs--; 533 ctx->cnt_merged.curbytes -= size; 534 malloc_mutex_unlock(&ctx->lock); 535 } 536 } 537} 538#endif 539 540#endif /* JEMALLOC_H_INLINES */ 541/******************************************************************************/ 542