prof.h revision 6c39f9e059d0825f4c29d8cec9f318b798912c3c
1cf2cfa174ca878c144e17e9fc60ca8e9070d7dededisonn@google.com/******************************************************************************/ 2cf2cfa174ca878c144e17e9fc60ca8e9070d7dededisonn@google.com#ifdef JEMALLOC_H_TYPES 3cf2cfa174ca878c144e17e9fc60ca8e9070d7dededisonn@google.com 4cf2cfa174ca878c144e17e9fc60ca8e9070d7dededisonn@google.comtypedef struct prof_bt_s prof_bt_t; 5cf2cfa174ca878c144e17e9fc60ca8e9070d7dededisonn@google.comtypedef struct prof_cnt_s prof_cnt_t; 6cf2cfa174ca878c144e17e9fc60ca8e9070d7dededisonn@google.comtypedef struct prof_thr_cnt_s prof_thr_cnt_t; 7cf2cfa174ca878c144e17e9fc60ca8e9070d7dededisonn@google.comtypedef struct prof_ctx_s prof_ctx_t; 8cf2cfa174ca878c144e17e9fc60ca8e9070d7dededisonn@google.comtypedef struct prof_tdata_s prof_tdata_t; 9cf2cfa174ca878c144e17e9fc60ca8e9070d7dededisonn@google.com 108cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com/* Option defaults. */ 118cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#ifdef JEMALLOC_PROF 128cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com# define PROF_PREFIX_DEFAULT "jeprof" 138cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#else 148cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com# define PROF_PREFIX_DEFAULT "" 158cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#endif 168cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#define LG_PROF_SAMPLE_DEFAULT 19 178cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#define LG_PROF_INTERVAL_DEFAULT -1 188cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 198cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com/* 208cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * Hard limit on stack backtrace depth. The version of prof_backtrace() that 218cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * is based on __builtin_return_address() necessarily has a hard-coded number 228cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * of backtrace frame handlers, and should be kept in sync with this setting. 238cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com */ 248cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#define PROF_BT_MAX 128 258cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 268cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com/* Maximum number of backtraces to store in each per thread LRU cache. */ 278cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#define PROF_TCMAX 1024 288cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 298cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com/* Initial hash table size. */ 308cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#define PROF_CKH_MINITEMS 64 318cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 328cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com/* Size of memory buffer to use when writing dump files. */ 338cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#define PROF_DUMP_BUFSIZE 65536 348cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 358cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com/* Size of stack-allocated buffer used by prof_printf(). */ 368cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#define PROF_PRINTF_BUFSIZE 128 378cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 388cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com/* 398cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * Number of mutexes shared among all ctx's. No space is allocated for these 408cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * unless profiling is enabled, so it's okay to over-provision. 418cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com */ 428cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#define PROF_NCTX_LOCKS 1024 438cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 448cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com/* 458cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * prof_tdata pointers close to NULL are used to encode state information that 468cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * is used for cleaning up during thread shutdown. 478cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com */ 488cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#define PROF_TDATA_STATE_REINCARNATED ((prof_tdata_t *)(uintptr_t)1) 498cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#define PROF_TDATA_STATE_PURGATORY ((prof_tdata_t *)(uintptr_t)2) 508cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#define PROF_TDATA_STATE_MAX PROF_TDATA_STATE_PURGATORY 518cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 528cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#endif /* JEMALLOC_H_TYPES */ 538cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com/******************************************************************************/ 548cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#ifdef JEMALLOC_H_STRUCTS 558cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 568cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comstruct prof_bt_s { 578cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* Backtrace, stored as len program counters. */ 588cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com void **vec; 598cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com unsigned len; 608cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com}; 618cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 628cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#ifdef JEMALLOC_PROF_LIBGCC 638cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com/* Data structure passed to libgcc _Unwind_Backtrace() callback functions. */ 648cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comtypedef struct { 658cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_bt_t *bt; 668cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com unsigned nignore; 678cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com unsigned max; 688cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com} prof_unwind_data_t; 698cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#endif 708cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 718cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comstruct prof_cnt_s { 728cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* 738cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * Profiling counters. An allocation/deallocation pair can operate on 748cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * different prof_thr_cnt_t objects that are linked into the same 758cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * prof_ctx_t cnts_ql, so it is possible for the cur* counters to go 768cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * negative. In principle it is possible for the *bytes counters to 778cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * overflow/underflow, but a general solution would require something 788cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * like 128-bit counters; this implementation doesn't bother to solve 798cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * that problem. 808cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com */ 818cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com int64_t curobjs; 828cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com int64_t curbytes; 838cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com uint64_t accumobjs; 848cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com uint64_t accumbytes; 858cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com}; 868cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 878cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comstruct prof_thr_cnt_s { 888cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* Linkage into prof_ctx_t's cnts_ql. */ 898cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com ql_elm(prof_thr_cnt_t) cnts_link; 908cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 918cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* Linkage into thread's LRU. */ 928cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com ql_elm(prof_thr_cnt_t) lru_link; 938cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 948cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* 958cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * Associated context. If a thread frees an object that it did not 968cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * allocate, it is possible that the context is not cached in the 978cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * thread's hash table, in which case it must be able to look up the 988cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * context, insert a new prof_thr_cnt_t into the thread's hash table, 998cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * and link it into the prof_ctx_t's cnts_ql. 1008cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com */ 1018cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_ctx_t *ctx; 1028cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 1038cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* 1048cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * Threads use memory barriers to update the counters. Since there is 1058cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * only ever one writer, the only challenge is for the reader to get a 1068cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * consistent read of the counters. 1078cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * 1088cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * The writer uses this series of operations: 1098cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * 1108cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * 1) Increment epoch to an odd number. 1118cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * 2) Update counters. 1128cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * 3) Increment epoch to an even number. 1138cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * 1148cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * The reader must assure 1) that the epoch is even while it reads the 1158cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * counters, and 2) that the epoch doesn't change between the time it 1168cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * starts and finishes reading the counters. 1178cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com */ 1188cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com unsigned epoch; 1198cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 1208cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* Profiling counters. */ 1218cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_cnt_t cnts; 1228cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com}; 1238cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 1248cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comstruct prof_ctx_s { 1258cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* Associated backtrace. */ 1268cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_bt_t *bt; 1278cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 1288cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* Protects nlimbo, cnt_merged, and cnts_ql. */ 1298cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com malloc_mutex_t *lock; 1308cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 1318cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* 1328cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * Number of threads that currently cause this ctx to be in a state of 1338cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * limbo due to one of: 1348cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * - Initializing per thread counters associated with this ctx. 1358cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * - Preparing to destroy this ctx. 1368cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * - Dumping a heap profile that includes this ctx. 1378cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * nlimbo must be 1 (single destroyer) in order to safely destroy the 1388cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * ctx. 1398cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com */ 1408cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com unsigned nlimbo; 1418cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 1428cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* Temporary storage for summation during dump. */ 1438cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_cnt_t cnt_summed; 1448cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 1458cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* When threads exit, they merge their stats into cnt_merged. */ 1468cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_cnt_t cnt_merged; 1478cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 1488cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* 1498cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * List of profile counters, one for each thread that has allocated in 1508cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * this context. 1518cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com */ 1528cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com ql_head(prof_thr_cnt_t) cnts_ql; 1538cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 1548cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* Linkage for list of contexts to be dumped. */ 1558cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com ql_elm(prof_ctx_t) dump_link; 1568cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com}; 1578cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comtypedef ql_head(prof_ctx_t) prof_ctx_list_t; 1588cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 1598cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comstruct prof_tdata_s { 1608cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* 1618cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * Hash of (prof_bt_t *)-->(prof_thr_cnt_t *). Each thread keeps a 1628cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * cache of backtraces, with associated thread-specific prof_thr_cnt_t 1638cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * objects. Other threads may read the prof_thr_cnt_t contents, but no 1648cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * others will ever write them. 1658cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * 1668cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * Upon thread exit, the thread must merge all the prof_thr_cnt_t 1678cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * counter data into the associated prof_ctx_t objects, and unlink/free 1688cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * the prof_thr_cnt_t objects. 1698cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com */ 1708cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com ckh_t bt2cnt; 1718cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 1728cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* LRU for contents of bt2cnt. */ 1738cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com ql_head(prof_thr_cnt_t) lru_ql; 1748cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 1758cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* Backtrace vector, used for calls to prof_backtrace(). */ 1768cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com void **vec; 1778cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 1788cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* Sampling state. */ 1798cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com uint64_t prng_state; 1808cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com uint64_t bytes_until_sample; 1818cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 1828cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* State used to avoid dumping while operating on prof internals. */ 1838cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com bool enq; 1848cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com bool enq_idump; 1858cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com bool enq_gdump; 1868cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com}; 1878cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 1888cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#endif /* JEMALLOC_H_STRUCTS */ 1898cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com/******************************************************************************/ 1908cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#ifdef JEMALLOC_H_EXTERNS 1918cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 1928cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comextern bool opt_prof; 1938cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com/* 1948cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * Even if opt_prof is true, sampling can be temporarily disabled by setting 1958cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * opt_prof_active to false. No locking is used when updating opt_prof_active, 1968cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * so there are no guarantees regarding how long it will take for all threads 1978cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * to notice state changes. 1988cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com */ 1998cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comextern bool opt_prof_active; 2008cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comextern size_t opt_lg_prof_sample; /* Mean bytes between samples. */ 2018cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comextern ssize_t opt_lg_prof_interval; /* lg(prof_interval). */ 2028cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comextern bool opt_prof_gdump; /* High-water memory dumping. */ 2038cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comextern bool opt_prof_final; /* Final profile dumping. */ 2048cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comextern bool opt_prof_leak; /* Dump leak summary at exit. */ 2058cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comextern bool opt_prof_accum; /* Report cumulative bytes. */ 2068cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comextern char opt_prof_prefix[ 2078cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* Minimize memory bloat for non-prof builds. */ 2088cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#ifdef JEMALLOC_PROF 2098cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com PATH_MAX + 2108cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#endif 2118cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 1]; 2128cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 2138cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com/* 2148cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * Profile dump interval, measured in bytes allocated. Each arena triggers a 2158cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * profile dump when it reaches this threshold. The effect is that the 2168cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * interval between profile dumps averages prof_interval, though the actual 2178cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * interval between dumps will tend to be sporadic, and the interval will be a 2188cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * maximum of approximately (prof_interval * narenas). 2198cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com */ 2208cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comextern uint64_t prof_interval; 2218cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 2228cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comvoid bt_init(prof_bt_t *bt, void **vec); 2238cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comvoid prof_backtrace(prof_bt_t *bt, unsigned nignore); 2248cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comprof_thr_cnt_t *prof_lookup(prof_bt_t *bt); 2258cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#ifdef JEMALLOC_JET 2268cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comsize_t prof_bt_count(void); 2278cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comtypedef int (prof_dump_open_t)(bool, const char *); 2288cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comextern prof_dump_open_t *prof_dump_open; 2298cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#endif 2308cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comvoid prof_idump(void); 2318cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.combool prof_mdump(const char *filename); 2328cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comvoid prof_gdump(void); 2338cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comprof_tdata_t *prof_tdata_init(void); 2348cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comvoid prof_tdata_cleanup(void *arg); 2358cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comvoid prof_boot0(void); 2368cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comvoid prof_boot1(void); 2378cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.combool prof_boot2(void); 2388cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comvoid prof_prefork(void); 2398cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comvoid prof_postfork_parent(void); 2408cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comvoid prof_postfork_child(void); 2418cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comvoid prof_sample_threshold_update(prof_tdata_t *prof_tdata); 2428cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 2438cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#endif /* JEMALLOC_H_EXTERNS */ 2448cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com/******************************************************************************/ 2458cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#ifdef JEMALLOC_H_INLINES 2468cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 2478cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#define PROF_ALLOC_PREP(nignore, size, ret) do { \ 2488cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_tdata_t *prof_tdata; \ 2498cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_bt_t bt; \ 2508cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com \ 2518cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com assert(size == s2u(size)); \ 2528cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com \ 2538cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com if (!opt_prof_active || \ 2548cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_sample_accum_update(size, false, &prof_tdata)) { \ 2558cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com ret = (prof_thr_cnt_t *)(uintptr_t)1U; \ 2568cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com } else { \ 2578cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com bt_init(&bt, prof_tdata->vec); \ 2588cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_backtrace(&bt, nignore); \ 2598cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com ret = prof_lookup(&bt); \ 2608cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com } \ 2618cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com} while (0) 2628cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 2638cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#ifndef JEMALLOC_ENABLE_INLINE 2648cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.commalloc_tsd_protos(JEMALLOC_ATTR(unused), prof_tdata, prof_tdata_t *) 2658cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 2668cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comprof_tdata_t *prof_tdata_get(bool create); 2678cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comvoid prof_sample_accum_update(size_t size, bool commit, 2688cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_tdata_t **prof_tdata_out); 2698cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comprof_ctx_t *prof_ctx_get(const void *ptr); 2708cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comvoid prof_ctx_set(const void *ptr, prof_ctx_t *ctx); 2718cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.combool prof_sample_accum_update(size_t size); 2728cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comvoid prof_malloc_record_object(const void *ptr, size_t usize, 2738cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_thr_cnt_t *cnt) 2748cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comvoid prof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt); 2758cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comvoid prof_realloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt, 2768cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com size_t old_usize, prof_ctx_t *old_ctx); 2778cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comvoid prof_free(const void *ptr, size_t size); 2788cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#endif 2798cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 2808cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_)) 2818cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com/* Thread-specific backtrace cache, used to reduce bt2ctx contention. */ 2828cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.commalloc_tsd_externs(prof_tdata, prof_tdata_t *) 2838cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.commalloc_tsd_funcs(JEMALLOC_INLINE, prof_tdata, prof_tdata_t *, NULL, 2848cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_tdata_cleanup) 2858cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 2868cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comJEMALLOC_INLINE prof_tdata_t * 2878cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comprof_tdata_get(bool create) 2888cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com{ 2898cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_tdata_t *prof_tdata; 2908cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 2918cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com cassert(config_prof); 2928cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 2938cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_tdata = *prof_tdata_tsd_get(); 2948cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com if (create && prof_tdata == NULL) 2958cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_tdata = prof_tdata_init(); 2968cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 2978cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com return (prof_tdata); 2988cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com} 2998cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 3008cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comJEMALLOC_INLINE prof_ctx_t * 3018cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comprof_ctx_get(const void *ptr) 3028cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com{ 3038cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_ctx_t *ret; 3048cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com arena_chunk_t *chunk; 3058cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 3068cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com cassert(config_prof); 3078cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com assert(ptr != NULL); 3088cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 3098cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 3108cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com if (chunk != ptr) { 3118cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* Region. */ 3128cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com ret = arena_prof_ctx_get(ptr); 3138cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com } else 3148cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com ret = huge_prof_ctx_get(ptr); 3158cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 3168cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com return (ret); 3178cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com} 3188cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 3198cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comJEMALLOC_INLINE void 3208cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comprof_ctx_set(const void *ptr, prof_ctx_t *ctx) 3218cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com{ 3228cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com arena_chunk_t *chunk; 3238cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 3248cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com cassert(config_prof); 3258cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com assert(ptr != NULL); 3268cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 3278cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 3288cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com if (chunk != ptr) { 3298cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* Region. */ 3308cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com arena_prof_ctx_set(ptr, ctx); 3318cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com } else 3328cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com huge_prof_ctx_set(ptr, ctx); 3338cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com} 3348cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 3358cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comJEMALLOC_INLINE bool 3368cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comprof_sample_accum_update(size_t size, bool commit, 3378cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_tdata_t **prof_tdata_out) 3388cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com{ 3398cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_tdata_t *prof_tdata; 3408cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 3418cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com cassert(config_prof); 3428cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 3438cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_tdata = prof_tdata_get(true); 3448cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) 3458cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_tdata = NULL; 3468cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 3478cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com if (prof_tdata_out != NULL) 3488cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com *prof_tdata_out = prof_tdata; 3498cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 3508cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com if (prof_tdata == NULL) 3518cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com return (true); 3528cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 3538cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com if (prof_tdata->bytes_until_sample >= size) { 3548cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com if (commit) 3558cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_tdata->bytes_until_sample -= size; 3568cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com return (true); 3578cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com } else { 3588cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* Compute new sample threshold. */ 3598cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com if (commit) 3608cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_sample_threshold_update(prof_tdata); 3618cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com return (false); 3628cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com } 3638cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com} 3648cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 3658cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comJEMALLOC_INLINE void 3668cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comprof_malloc_record_object(const void *ptr, size_t usize, prof_thr_cnt_t *cnt) { 3678cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_ctx_set(ptr, cnt->ctx); 3688cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 3698cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com cnt->epoch++; 3708cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /*********/ 3718cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com mb_write(); 3728cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /*********/ 3738cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com cnt->cnts.curobjs++; 3748cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com cnt->cnts.curbytes += usize; 3758cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com if (opt_prof_accum) { 3768cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com cnt->cnts.accumobjs++; 3778cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com cnt->cnts.accumbytes += usize; 3788cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com } 3798cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /*********/ 3808cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com mb_write(); 3818cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /*********/ 3828cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com cnt->epoch++; 3838cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /*********/ 3848cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com mb_write(); 3858cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /*********/ 3868cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com} 3878cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 3888cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comJEMALLOC_INLINE void 3898cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comprof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt) 3908cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com{ 3918cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 3928cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com cassert(config_prof); 3938cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com assert(ptr != NULL); 3948cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com assert(usize == isalloc(ptr, true)); 3958cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 3968cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com if (prof_sample_accum_update(usize, true, NULL)) { 3978cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* 3988cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * Don't sample. For malloc()-like allocation, it is 3998cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * always possible to tell in advance how large an 4008cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * object's usable size will be, so there should never 4018cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * be a difference between the usize passed to 4028cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * PROF_ALLOC_PREP() and prof_malloc(). 4038cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com */ 4048cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com assert((uintptr_t)cnt == (uintptr_t)1U); 4058cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com } 4068cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 4078cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com if ((uintptr_t)cnt > (uintptr_t)1U) 4088cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_malloc_record_object(ptr, usize, cnt); 4098cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com else 4108cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U); 4118cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com} 4128cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 4138cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comJEMALLOC_INLINE void 4148cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comprof_realloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt, 4158cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com size_t old_usize, prof_ctx_t *old_ctx) 4168cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com{ 4178cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_thr_cnt_t *told_cnt; 4188cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 4198cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com cassert(config_prof); 4208cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com assert(ptr != NULL || (uintptr_t)cnt <= (uintptr_t)1U); 4218cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 4228cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com if (ptr != NULL) { 4238cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com assert(usize == isalloc(ptr, true)); 4248cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com if (prof_sample_accum_update(usize, true, NULL)) { 4258cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* 4268cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * Don't sample. The usize passed to 4278cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * PROF_ALLOC_PREP() was larger than what 4288cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * actually got allocated, so a backtrace was 4298cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * captured for this allocation, even though 4308cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * its actual usize was insufficient to cross 4318cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * the sample threshold. 4328cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com */ 4338cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com cnt = (prof_thr_cnt_t *)(uintptr_t)1U; 4348cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com } 4358cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com } 4368cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 4378cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com if ((uintptr_t)old_ctx > (uintptr_t)1U) { 4388cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com told_cnt = prof_lookup(old_ctx->bt); 4398cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com if (told_cnt == NULL) { 4408cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* 4418cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * It's too late to propagate OOM for this realloc(), 4428cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * so operate directly on old_cnt->ctx->cnt_merged. 4438cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com */ 4448cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com malloc_mutex_lock(old_ctx->lock); 4458cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com old_ctx->cnt_merged.curobjs--; 4468cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com old_ctx->cnt_merged.curbytes -= old_usize; 4478cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com malloc_mutex_unlock(old_ctx->lock); 4488cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com told_cnt = (prof_thr_cnt_t *)(uintptr_t)1U; 4498cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com } 4508cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com } else 4518cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com told_cnt = (prof_thr_cnt_t *)(uintptr_t)1U; 4528cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 4538cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com if ((uintptr_t)told_cnt > (uintptr_t)1U) 4548cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com told_cnt->epoch++; 4558cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com if ((uintptr_t)cnt > (uintptr_t)1U) { 4568cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_ctx_set(ptr, cnt->ctx); 4578cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com cnt->epoch++; 4588cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com } else if (ptr != NULL) 4598cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U); 4608cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /*********/ 4618cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com mb_write(); 4628cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /*********/ 4638cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com if ((uintptr_t)told_cnt > (uintptr_t)1U) { 4648cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com told_cnt->cnts.curobjs--; 4658cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com told_cnt->cnts.curbytes -= old_usize; 4668cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com } 4678cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com if ((uintptr_t)cnt > (uintptr_t)1U) { 4688cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com cnt->cnts.curobjs++; 4698cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com cnt->cnts.curbytes += usize; 4708cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com if (opt_prof_accum) { 4718cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com cnt->cnts.accumobjs++; 4728cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com cnt->cnts.accumbytes += usize; 4738cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com } 4748cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com } 4758cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /*********/ 4768cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com mb_write(); 4778cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /*********/ 4788cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com if ((uintptr_t)told_cnt > (uintptr_t)1U) 4798cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com told_cnt->epoch++; 4808cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com if ((uintptr_t)cnt > (uintptr_t)1U) 4818cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com cnt->epoch++; 4828cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /*********/ 4838cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com mb_write(); /* Not strictly necessary. */ 4848cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com} 4858cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 4868cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comJEMALLOC_INLINE void 4878cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.comprof_free(const void *ptr, size_t size) 4888cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com{ 4898cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_ctx_t *ctx = prof_ctx_get(ptr); 4908cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 4918cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com cassert(config_prof); 4928cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 4938cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com if ((uintptr_t)ctx > (uintptr_t)1) { 4948cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com prof_thr_cnt_t *tcnt; 4958cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com assert(size == isalloc(ptr, true)); 4968cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com tcnt = prof_lookup(ctx->bt); 4978cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 4988cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com if (tcnt != NULL) { 4998cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com tcnt->epoch++; 5008cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /*********/ 5018cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com mb_write(); 5028cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /*********/ 5038cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com tcnt->cnts.curobjs--; 5048cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com tcnt->cnts.curbytes -= size; 5058cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /*********/ 5068cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com mb_write(); 5078cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /*********/ 5088cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com tcnt->epoch++; 5098cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /*********/ 5108cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com mb_write(); 5118cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /*********/ 5128cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com } else { 5138cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com /* 5148cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * OOM during free() cannot be propagated, so operate 5158cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com * directly on cnt->ctx->cnt_merged. 5168cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com */ 5178cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com malloc_mutex_lock(ctx->lock); 5188cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com ctx->cnt_merged.curobjs--; 5198cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com ctx->cnt_merged.curbytes -= size; 5208cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com malloc_mutex_unlock(ctx->lock); 521cf2cfa174ca878c144e17e9fc60ca8e9070d7dededisonn@google.com } 5228cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com } 5238cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com} 524cf2cfa174ca878c144e17e9fc60ca8e9070d7dededisonn@google.com#endif 5258cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com 5268cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com#endif /* JEMALLOC_H_INLINES */ 5278cee797901763ab0922eb9ef484cfdcbc94bee54edisonn@google.com/******************************************************************************/ 528cf2cfa174ca878c144e17e9fc60ca8e9070d7dededisonn@google.com