16109fe07a14b7a619365977d9523db9f8b333792Jason Evans/******************************************************************************/ 26109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_H_TYPES 36109fe07a14b7a619365977d9523db9f8b333792Jason Evans 46109fe07a14b7a619365977d9523db9f8b333792Jason Evanstypedef struct prof_bt_s prof_bt_t; 56109fe07a14b7a619365977d9523db9f8b333792Jason Evanstypedef struct prof_cnt_s prof_cnt_t; 66109fe07a14b7a619365977d9523db9f8b333792Jason Evanstypedef struct prof_thr_cnt_s prof_thr_cnt_t; 76109fe07a14b7a619365977d9523db9f8b333792Jason Evanstypedef struct prof_ctx_s prof_ctx_t; 84d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evanstypedef struct prof_tdata_s prof_tdata_t; 96109fe07a14b7a619365977d9523db9f8b333792Jason Evans 10b9477e782b07afa38c4b1dc0688e053be8a84dd8Jason Evans/* Option defaults. */ 115f60afa01eb2cf7d44024d162a1ecc6cceedcca1Jason Evans#ifdef JEMALLOC_PROF 125f60afa01eb2cf7d44024d162a1ecc6cceedcca1Jason Evans# define PROF_PREFIX_DEFAULT "jeprof" 135f60afa01eb2cf7d44024d162a1ecc6cceedcca1Jason Evans#else 145f60afa01eb2cf7d44024d162a1ecc6cceedcca1Jason Evans# define PROF_PREFIX_DEFAULT "" 155f60afa01eb2cf7d44024d162a1ecc6cceedcca1Jason Evans#endif 160b25fe79aaf8840a5acda7e3160a053d42349872Jason Evans#define LG_PROF_SAMPLE_DEFAULT 19 1737dab02e524c14cffa4bb938242e4cf2695a4a7dJason Evans#define LG_PROF_INTERVAL_DEFAULT -1 180b526ff94da7e59aa947a4d3529b2376794f8b01Jason Evans 196109fe07a14b7a619365977d9523db9f8b333792Jason Evans/* 205389146191b279ca3b90028357dd6ad66b283defJason Evans * Hard limit on stack backtrace depth. The version of prof_backtrace() that 215389146191b279ca3b90028357dd6ad66b283defJason Evans * is based on __builtin_return_address() necessarily has a hard-coded number 225389146191b279ca3b90028357dd6ad66b283defJason Evans * of backtrace frame handlers, and should be kept in sync with this setting. 236109fe07a14b7a619365977d9523db9f8b333792Jason Evans */ 245389146191b279ca3b90028357dd6ad66b283defJason Evans#define PROF_BT_MAX 128 255389146191b279ca3b90028357dd6ad66b283defJason Evans 265389146191b279ca3b90028357dd6ad66b283defJason Evans/* Maximum number of backtraces to store in each per thread LRU cache. */ 275389146191b279ca3b90028357dd6ad66b283defJason Evans#define PROF_TCMAX 1024 286109fe07a14b7a619365977d9523db9f8b333792Jason Evans 296109fe07a14b7a619365977d9523db9f8b333792Jason Evans/* Initial hash table size. */ 30cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans#define PROF_CKH_MINITEMS 64 316109fe07a14b7a619365977d9523db9f8b333792Jason Evans 326109fe07a14b7a619365977d9523db9f8b333792Jason Evans/* Size of memory buffer to use when writing dump files. */ 33cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans#define PROF_DUMP_BUFSIZE 65536 34cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans 35cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans/* Size of stack-allocated buffer used by prof_printf(). */ 36cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans#define PROF_PRINTF_BUFSIZE 128 376109fe07a14b7a619365977d9523db9f8b333792Jason Evans 386da5418ded9170b087c35960e0010006430117c1Jason Evans/* 396da5418ded9170b087c35960e0010006430117c1Jason Evans * Number of mutexes shared among all ctx's. No space is allocated for these 406da5418ded9170b087c35960e0010006430117c1Jason Evans * unless profiling is enabled, so it's okay to over-provision. 416da5418ded9170b087c35960e0010006430117c1Jason Evans */ 426da5418ded9170b087c35960e0010006430117c1Jason Evans#define PROF_NCTX_LOCKS 1024 436da5418ded9170b087c35960e0010006430117c1Jason Evans 440050a0f7e6ea5a33c9aed769e2652afe20714194Jason Evans/* 450050a0f7e6ea5a33c9aed769e2652afe20714194Jason Evans * prof_tdata pointers close to NULL are used to encode state information that 460050a0f7e6ea5a33c9aed769e2652afe20714194Jason Evans * is used for cleaning up during thread shutdown. 470050a0f7e6ea5a33c9aed769e2652afe20714194Jason Evans */ 480050a0f7e6ea5a33c9aed769e2652afe20714194Jason Evans#define PROF_TDATA_STATE_REINCARNATED ((prof_tdata_t *)(uintptr_t)1) 490050a0f7e6ea5a33c9aed769e2652afe20714194Jason Evans#define PROF_TDATA_STATE_PURGATORY ((prof_tdata_t *)(uintptr_t)2) 500050a0f7e6ea5a33c9aed769e2652afe20714194Jason Evans#define PROF_TDATA_STATE_MAX PROF_TDATA_STATE_PURGATORY 510050a0f7e6ea5a33c9aed769e2652afe20714194Jason Evans 526109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif /* JEMALLOC_H_TYPES */ 536109fe07a14b7a619365977d9523db9f8b333792Jason Evans/******************************************************************************/ 546109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_H_STRUCTS 556109fe07a14b7a619365977d9523db9f8b333792Jason Evans 566109fe07a14b7a619365977d9523db9f8b333792Jason Evansstruct prof_bt_s { 576109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* Backtrace, stored as len program counters. */ 584d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans void **vec; 594d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans unsigned len; 606109fe07a14b7a619365977d9523db9f8b333792Jason Evans}; 616109fe07a14b7a619365977d9523db9f8b333792Jason Evans 62b27805b36309681da1936eb33044584547552340Jason Evans#ifdef JEMALLOC_PROF_LIBGCC 63b27805b36309681da1936eb33044584547552340Jason Evans/* Data structure passed to libgcc _Unwind_Backtrace() callback functions. */ 64b27805b36309681da1936eb33044584547552340Jason Evanstypedef struct { 65a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans prof_bt_t *bt; 66a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans unsigned max; 67b27805b36309681da1936eb33044584547552340Jason Evans} prof_unwind_data_t; 68b27805b36309681da1936eb33044584547552340Jason Evans#endif 69b27805b36309681da1936eb33044584547552340Jason Evans 706109fe07a14b7a619365977d9523db9f8b333792Jason Evansstruct prof_cnt_s { 716109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* 726109fe07a14b7a619365977d9523db9f8b333792Jason Evans * Profiling counters. An allocation/deallocation pair can operate on 736109fe07a14b7a619365977d9523db9f8b333792Jason Evans * different prof_thr_cnt_t objects that are linked into the same 74a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans * prof_ctx_t cnts_ql, so it is possible for the cur* counters to go 756109fe07a14b7a619365977d9523db9f8b333792Jason Evans * negative. In principle it is possible for the *bytes counters to 76a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans * overflow/underflow, but a general solution would require something 77a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans * like 128-bit counters; this implementation doesn't bother to solve 78a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans * that problem. 796109fe07a14b7a619365977d9523db9f8b333792Jason Evans */ 806109fe07a14b7a619365977d9523db9f8b333792Jason Evans int64_t curobjs; 816109fe07a14b7a619365977d9523db9f8b333792Jason Evans int64_t curbytes; 826109fe07a14b7a619365977d9523db9f8b333792Jason Evans uint64_t accumobjs; 836109fe07a14b7a619365977d9523db9f8b333792Jason Evans uint64_t accumbytes; 846109fe07a14b7a619365977d9523db9f8b333792Jason Evans}; 856109fe07a14b7a619365977d9523db9f8b333792Jason Evans 866109fe07a14b7a619365977d9523db9f8b333792Jason Evansstruct prof_thr_cnt_s { 87a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans /* Linkage into prof_ctx_t's cnts_ql. */ 88a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans ql_elm(prof_thr_cnt_t) cnts_link; 89a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans 90a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans /* Linkage into thread's LRU. */ 91a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans ql_elm(prof_thr_cnt_t) lru_link; 926109fe07a14b7a619365977d9523db9f8b333792Jason Evans 936109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* 946109fe07a14b7a619365977d9523db9f8b333792Jason Evans * Associated context. If a thread frees an object that it did not 956109fe07a14b7a619365977d9523db9f8b333792Jason Evans * allocate, it is possible that the context is not cached in the 966109fe07a14b7a619365977d9523db9f8b333792Jason Evans * thread's hash table, in which case it must be able to look up the 976109fe07a14b7a619365977d9523db9f8b333792Jason Evans * context, insert a new prof_thr_cnt_t into the thread's hash table, 98a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans * and link it into the prof_ctx_t's cnts_ql. 996109fe07a14b7a619365977d9523db9f8b333792Jason Evans */ 1006109fe07a14b7a619365977d9523db9f8b333792Jason Evans prof_ctx_t *ctx; 1016109fe07a14b7a619365977d9523db9f8b333792Jason Evans 1026109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* 1036109fe07a14b7a619365977d9523db9f8b333792Jason Evans * Threads use memory barriers to update the counters. Since there is 1046109fe07a14b7a619365977d9523db9f8b333792Jason Evans * only ever one writer, the only challenge is for the reader to get a 1056109fe07a14b7a619365977d9523db9f8b333792Jason Evans * consistent read of the counters. 1066109fe07a14b7a619365977d9523db9f8b333792Jason Evans * 1076109fe07a14b7a619365977d9523db9f8b333792Jason Evans * The writer uses this series of operations: 1086109fe07a14b7a619365977d9523db9f8b333792Jason Evans * 1096109fe07a14b7a619365977d9523db9f8b333792Jason Evans * 1) Increment epoch to an odd number. 1106109fe07a14b7a619365977d9523db9f8b333792Jason Evans * 2) Update counters. 1116109fe07a14b7a619365977d9523db9f8b333792Jason Evans * 3) Increment epoch to an even number. 1126109fe07a14b7a619365977d9523db9f8b333792Jason Evans * 1136109fe07a14b7a619365977d9523db9f8b333792Jason Evans * The reader must assure 1) that the epoch is even while it reads the 1146109fe07a14b7a619365977d9523db9f8b333792Jason Evans * counters, and 2) that the epoch doesn't change between the time it 1156109fe07a14b7a619365977d9523db9f8b333792Jason Evans * starts and finishes reading the counters. 1166109fe07a14b7a619365977d9523db9f8b333792Jason Evans */ 1176109fe07a14b7a619365977d9523db9f8b333792Jason Evans unsigned epoch; 1186109fe07a14b7a619365977d9523db9f8b333792Jason Evans 1196109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* Profiling counters. */ 1206109fe07a14b7a619365977d9523db9f8b333792Jason Evans prof_cnt_t cnts; 1216109fe07a14b7a619365977d9523db9f8b333792Jason Evans}; 1226109fe07a14b7a619365977d9523db9f8b333792Jason Evans 1236109fe07a14b7a619365977d9523db9f8b333792Jason Evansstruct prof_ctx_s { 1245065156f3f90e421ba2b1a914e47eeb30d83d994Jason Evans /* Associated backtrace. */ 1255065156f3f90e421ba2b1a914e47eeb30d83d994Jason Evans prof_bt_t *bt; 1265065156f3f90e421ba2b1a914e47eeb30d83d994Jason Evans 12752386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans /* Protects nlimbo, cnt_merged, and cnts_ql. */ 1286da5418ded9170b087c35960e0010006430117c1Jason Evans malloc_mutex_t *lock; 1296109fe07a14b7a619365977d9523db9f8b333792Jason Evans 13052386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans /* 13152386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans * Number of threads that currently cause this ctx to be in a state of 13252386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans * limbo due to one of: 13352386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans * - Initializing per thread counters associated with this ctx. 13452386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans * - Preparing to destroy this ctx. 1354f37ef693e3d5903ce07dc0b61c0da320b35e3d9Jason Evans * - Dumping a heap profile that includes this ctx. 13652386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans * nlimbo must be 1 (single destroyer) in order to safely destroy the 13752386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans * ctx. 13852386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans */ 13952386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans unsigned nlimbo; 14052386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans 141a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans /* Temporary storage for summation during dump. */ 142a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans prof_cnt_t cnt_summed; 1436109fe07a14b7a619365977d9523db9f8b333792Jason Evans 1446109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* When threads exit, they merge their stats into cnt_merged. */ 1456109fe07a14b7a619365977d9523db9f8b333792Jason Evans prof_cnt_t cnt_merged; 1466109fe07a14b7a619365977d9523db9f8b333792Jason Evans 1476109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* 1486109fe07a14b7a619365977d9523db9f8b333792Jason Evans * List of profile counters, one for each thread that has allocated in 1496109fe07a14b7a619365977d9523db9f8b333792Jason Evans * this context. 1506109fe07a14b7a619365977d9523db9f8b333792Jason Evans */ 1516109fe07a14b7a619365977d9523db9f8b333792Jason Evans ql_head(prof_thr_cnt_t) cnts_ql; 1524f37ef693e3d5903ce07dc0b61c0da320b35e3d9Jason Evans 1534f37ef693e3d5903ce07dc0b61c0da320b35e3d9Jason Evans /* Linkage for list of contexts to be dumped. */ 1544f37ef693e3d5903ce07dc0b61c0da320b35e3d9Jason Evans ql_elm(prof_ctx_t) dump_link; 1556109fe07a14b7a619365977d9523db9f8b333792Jason Evans}; 1564f37ef693e3d5903ce07dc0b61c0da320b35e3d9Jason Evanstypedef ql_head(prof_ctx_t) prof_ctx_list_t; 1576109fe07a14b7a619365977d9523db9f8b333792Jason Evans 1584d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evansstruct prof_tdata_s { 1594d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans /* 1604d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans * Hash of (prof_bt_t *)-->(prof_thr_cnt_t *). Each thread keeps a 1614d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans * cache of backtraces, with associated thread-specific prof_thr_cnt_t 1624d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans * objects. Other threads may read the prof_thr_cnt_t contents, but no 1634d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans * others will ever write them. 1644d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans * 1654d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans * Upon thread exit, the thread must merge all the prof_thr_cnt_t 1664d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans * counter data into the associated prof_ctx_t objects, and unlink/free 1674d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans * the prof_thr_cnt_t objects. 1684d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans */ 169a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans ckh_t bt2cnt; 170a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans 171a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans /* LRU for contents of bt2cnt. */ 172a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans ql_head(prof_thr_cnt_t) lru_ql; 1734d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 1744d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans /* Backtrace vector, used for calls to prof_backtrace(). */ 1754d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans void **vec; 1764d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 1774d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans /* Sampling state. */ 17884f7cdb0c588322dfd50a26497fc1cb54b792018Jason Evans uint64_t prng_state; 1796c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer uint64_t bytes_until_sample; 18052386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans 18152386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans /* State used to avoid dumping while operating on prof internals. */ 18252386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans bool enq; 18352386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans bool enq_idump; 18452386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans bool enq_gdump; 185a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans}; 186a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans 1876109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif /* JEMALLOC_H_STRUCTS */ 1886109fe07a14b7a619365977d9523db9f8b333792Jason Evans/******************************************************************************/ 1896109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_H_EXTERNS 1906109fe07a14b7a619365977d9523db9f8b333792Jason Evans 1916109fe07a14b7a619365977d9523db9f8b333792Jason Evansextern bool opt_prof; 192f18c98200145de70779a1b3286e7829b0268231eJason Evans/* 193f18c98200145de70779a1b3286e7829b0268231eJason Evans * Even if opt_prof is true, sampling can be temporarily disabled by setting 194f18c98200145de70779a1b3286e7829b0268231eJason Evans * opt_prof_active to false. No locking is used when updating opt_prof_active, 195f18c98200145de70779a1b3286e7829b0268231eJason Evans * so there are no guarantees regarding how long it will take for all threads 196f18c98200145de70779a1b3286e7829b0268231eJason Evans * to notice state changes. 197f18c98200145de70779a1b3286e7829b0268231eJason Evans */ 198f18c98200145de70779a1b3286e7829b0268231eJason Evansextern bool opt_prof_active; 199a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evansextern size_t opt_lg_prof_sample; /* Mean bytes between samples. */ 200a02fc08ec9dd8479a6430155b6a433da09f6ff10Jason Evansextern ssize_t opt_lg_prof_interval; /* lg(prof_interval). */ 201e73397062ac3ab28a9d377591b63ed19fd154ccaJason Evansextern bool opt_prof_gdump; /* High-water memory dumping. */ 2020b25fe79aaf8840a5acda7e3160a053d42349872Jason Evansextern bool opt_prof_final; /* Final profile dumping. */ 203a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evansextern bool opt_prof_leak; /* Dump leak summary at exit. */ 204a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evansextern bool opt_prof_accum; /* Report cumulative bytes. */ 205eefdd02e70ec1b9cf11920fcff585835dcbd766bJason Evansextern char opt_prof_prefix[ 206eefdd02e70ec1b9cf11920fcff585835dcbd766bJason Evans /* Minimize memory bloat for non-prof builds. */ 207eefdd02e70ec1b9cf11920fcff585835dcbd766bJason Evans#ifdef JEMALLOC_PROF 208eefdd02e70ec1b9cf11920fcff585835dcbd766bJason Evans PATH_MAX + 209eefdd02e70ec1b9cf11920fcff585835dcbd766bJason Evans#endif 210eefdd02e70ec1b9cf11920fcff585835dcbd766bJason Evans 1]; 2116109fe07a14b7a619365977d9523db9f8b333792Jason Evans 212d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans/* 213d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans * Profile dump interval, measured in bytes allocated. Each arena triggers a 214d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans * profile dump when it reaches this threshold. The effect is that the 215d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans * interval between profile dumps averages prof_interval, though the actual 216d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans * interval between dumps will tend to be sporadic, and the interval will be a 217d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans * maximum of approximately (prof_interval * narenas). 218d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans */ 219d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evansextern uint64_t prof_interval; 220d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans 2214d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evansvoid bt_init(prof_bt_t *bt, void **vec); 22273b37a9697acd53496bbef06ed25696e0c897341Jason Evansvoid prof_backtrace(prof_bt_t *bt); 2234d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evansprof_thr_cnt_t *prof_lookup(prof_bt_t *bt); 224772163b4f3d8e9a12343e9215f6b070068507604Jason Evans#ifdef JEMALLOC_JET 225772163b4f3d8e9a12343e9215f6b070068507604Jason Evanssize_t prof_bt_count(void); 226772163b4f3d8e9a12343e9215f6b070068507604Jason Evanstypedef int (prof_dump_open_t)(bool, const char *); 227772163b4f3d8e9a12343e9215f6b070068507604Jason Evansextern prof_dump_open_t *prof_dump_open; 228772163b4f3d8e9a12343e9215f6b070068507604Jason Evans#endif 2296109fe07a14b7a619365977d9523db9f8b333792Jason Evansvoid prof_idump(void); 23022ca855e8fbf4c592f5e46aaec381b68de60c71aJason Evansbool prof_mdump(const char *filename); 231e73397062ac3ab28a9d377591b63ed19fd154ccaJason Evansvoid prof_gdump(void); 2324d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evansprof_tdata_t *prof_tdata_init(void); 233cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evansvoid prof_tdata_cleanup(void *arg); 2346109fe07a14b7a619365977d9523db9f8b333792Jason Evansvoid prof_boot0(void); 235e73397062ac3ab28a9d377591b63ed19fd154ccaJason Evansvoid prof_boot1(void); 236e73397062ac3ab28a9d377591b63ed19fd154ccaJason Evansbool prof_boot2(void); 23720f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evansvoid prof_prefork(void); 23820f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evansvoid prof_postfork_parent(void); 23920f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evansvoid prof_postfork_child(void); 2406c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurervoid prof_sample_threshold_update(prof_tdata_t *prof_tdata); 2416109fe07a14b7a619365977d9523db9f8b333792Jason Evans 2426109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif /* JEMALLOC_H_EXTERNS */ 2436109fe07a14b7a619365977d9523db9f8b333792Jason Evans/******************************************************************************/ 2446109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_H_INLINES 2456109fe07a14b7a619365977d9523db9f8b333792Jason Evans 24673b37a9697acd53496bbef06ed25696e0c897341Jason Evans#define PROF_ALLOC_PREP(size, ret) do { \ 247a507004d294ad0c78b4d01559479620ebb272a49Jason Evans prof_tdata_t *prof_tdata; \ 248a507004d294ad0c78b4d01559479620ebb272a49Jason Evans prof_bt_t bt; \ 249a507004d294ad0c78b4d01559479620ebb272a49Jason Evans \ 250a507004d294ad0c78b4d01559479620ebb272a49Jason Evans assert(size == s2u(size)); \ 251a507004d294ad0c78b4d01559479620ebb272a49Jason Evans \ 2526c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer if (!opt_prof_active || \ 2536c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer prof_sample_accum_update(size, false, &prof_tdata)) { \ 254a507004d294ad0c78b4d01559479620ebb272a49Jason Evans ret = (prof_thr_cnt_t *)(uintptr_t)1U; \ 2556c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer } else { \ 256a507004d294ad0c78b4d01559479620ebb272a49Jason Evans bt_init(&bt, prof_tdata->vec); \ 25773b37a9697acd53496bbef06ed25696e0c897341Jason Evans prof_backtrace(&bt); \ 258a507004d294ad0c78b4d01559479620ebb272a49Jason Evans ret = prof_lookup(&bt); \ 259a507004d294ad0c78b4d01559479620ebb272a49Jason Evans } \ 260a507004d294ad0c78b4d01559479620ebb272a49Jason Evans} while (0) 261a507004d294ad0c78b4d01559479620ebb272a49Jason Evans 2624d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans#ifndef JEMALLOC_ENABLE_INLINE 263cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evansmalloc_tsd_protos(JEMALLOC_ATTR(unused), prof_tdata, prof_tdata_t *) 264cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans 265bbe29d374d0fa5f4684621f16c099294e56c26efJason Evansprof_tdata_t *prof_tdata_get(bool create); 2660b49403958b68294eee0eca8a0b5195e761cf316Jason Evansbool prof_sample_accum_update(size_t size, bool commit, 2676c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer prof_tdata_t **prof_tdata_out); 2684d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evansprof_ctx_t *prof_ctx_get(const void *ptr); 2699b0cbf0850b130a9b0a8c58bd10b2926b2083510Jason Evansvoid prof_ctx_set(const void *ptr, prof_ctx_t *ctx); 2706c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurervoid prof_malloc_record_object(const void *ptr, size_t usize, 2710b49403958b68294eee0eca8a0b5195e761cf316Jason Evans prof_thr_cnt_t *cnt); 272665769357cd77b74e00a146f196fff19243b33c4Jason Evansvoid prof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt); 273665769357cd77b74e00a146f196fff19243b33c4Jason Evansvoid prof_realloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt, 274665769357cd77b74e00a146f196fff19243b33c4Jason Evans size_t old_usize, prof_ctx_t *old_ctx); 275e4f7846f1fd279a039ffa2a41707348187219de4Jason Evansvoid prof_free(const void *ptr, size_t size); 2764d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans#endif 2774d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 2784d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_)) 279cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans/* Thread-specific backtrace cache, used to reduce bt2ctx contention. */ 280cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evansmalloc_tsd_externs(prof_tdata, prof_tdata_t *) 281cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evansmalloc_tsd_funcs(JEMALLOC_INLINE, prof_tdata, prof_tdata_t *, NULL, 282cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans prof_tdata_cleanup) 283cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans 28452386b2dc689db3bf71307424c4e1a2b7044c363Jason EvansJEMALLOC_INLINE prof_tdata_t * 285bbe29d374d0fa5f4684621f16c099294e56c26efJason Evansprof_tdata_get(bool create) 28652386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans{ 28752386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans prof_tdata_t *prof_tdata; 28852386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans 28952386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans cassert(config_prof); 29052386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans 29152386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans prof_tdata = *prof_tdata_tsd_get(); 292bbe29d374d0fa5f4684621f16c099294e56c26efJason Evans if (create && prof_tdata == NULL) 293bbe29d374d0fa5f4684621f16c099294e56c26efJason Evans prof_tdata = prof_tdata_init(); 29452386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans 29552386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans return (prof_tdata); 29652386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans} 29752386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans 2984d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason EvansJEMALLOC_INLINE prof_ctx_t * 2994d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evansprof_ctx_get(const void *ptr) 3004d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans{ 3014d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans prof_ctx_t *ret; 3024d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans arena_chunk_t *chunk; 3034d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 3047372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 3054d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans assert(ptr != NULL); 3064d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 3074d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 3084d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans if (chunk != ptr) { 3094d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans /* Region. */ 3104d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans ret = arena_prof_ctx_get(ptr); 3114d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans } else 3124d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans ret = huge_prof_ctx_get(ptr); 3134d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 3144d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans return (ret); 3154d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans} 3164d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 3174d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason EvansJEMALLOC_INLINE void 3189b0cbf0850b130a9b0a8c58bd10b2926b2083510Jason Evansprof_ctx_set(const void *ptr, prof_ctx_t *ctx) 3194d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans{ 3204d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans arena_chunk_t *chunk; 3214d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 3227372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 3234d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans assert(ptr != NULL); 3244d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 3254d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 3264d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans if (chunk != ptr) { 3274d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans /* Region. */ 3289b0cbf0850b130a9b0a8c58bd10b2926b2083510Jason Evans arena_prof_ctx_set(ptr, ctx); 3294d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans } else 3304d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans huge_prof_ctx_set(ptr, ctx); 3314d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans} 3324d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 3334d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason EvansJEMALLOC_INLINE bool 3346c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurerprof_sample_accum_update(size_t size, bool commit, 3356c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer prof_tdata_t **prof_tdata_out) 3364d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans{ 3374d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans prof_tdata_t *prof_tdata; 3384d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 3397372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 3404d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 3416c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer prof_tdata = prof_tdata_get(true); 342f27899402914065a6c1484ea8d81a2c8b70aa659Jason Evans if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) 3436c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer prof_tdata = NULL; 3446c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer 3456c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer if (prof_tdata_out != NULL) 3466c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer *prof_tdata_out = prof_tdata; 3476c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer 3486c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer if (prof_tdata == NULL) 349f27899402914065a6c1484ea8d81a2c8b70aa659Jason Evans return (true); 3504d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 3516c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer if (prof_tdata->bytes_until_sample >= size) { 3526c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer if (commit) 3536c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer prof_tdata->bytes_until_sample -= size; 3546c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer return (true); 3556c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer } else { 356ff7450727f64180367f430b1b747f9e682e26df4Jason Evans /* Compute new sample threshold. */ 3576c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer if (commit) 3584d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans prof_sample_threshold_update(prof_tdata); 3594d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans return (false); 3604d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans } 3614d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans} 3624d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 3634d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason EvansJEMALLOC_INLINE void 3646c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurerprof_malloc_record_object(const void *ptr, size_t usize, prof_thr_cnt_t *cnt) { 3656c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer prof_ctx_set(ptr, cnt->ctx); 3666c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer 3676c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer cnt->epoch++; 3686c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer /*********/ 3696c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer mb_write(); 3706c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer /*********/ 3716c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer cnt->cnts.curobjs++; 3726c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer cnt->cnts.curbytes += usize; 3736c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer if (opt_prof_accum) { 3746c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer cnt->cnts.accumobjs++; 3756c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer cnt->cnts.accumbytes += usize; 3766c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer } 3776c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer /*********/ 3786c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer mb_write(); 3796c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer /*********/ 3806c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer cnt->epoch++; 3816c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer /*********/ 3826c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer mb_write(); 3836c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer /*********/ 3846c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer} 3856c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer 3866c39f9e059d0825f4c29d8cec9f318b798912c3cBen MaurerJEMALLOC_INLINE void 387665769357cd77b74e00a146f196fff19243b33c4Jason Evansprof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt) 3884d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans{ 3894d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 3907372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 3914d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans assert(ptr != NULL); 392665769357cd77b74e00a146f196fff19243b33c4Jason Evans assert(usize == isalloc(ptr, true)); 3934d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 3946c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer if (prof_sample_accum_update(usize, true, NULL)) { 3956c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer /* 3966c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer * Don't sample. For malloc()-like allocation, it is 3976c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer * always possible to tell in advance how large an 3986c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer * object's usable size will be, so there should never 3996c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer * be a difference between the usize passed to 4006c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer * PROF_ALLOC_PREP() and prof_malloc(). 4016c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer */ 4026c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer assert((uintptr_t)cnt == (uintptr_t)1U); 4034d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans } 4044d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 4056c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer if ((uintptr_t)cnt > (uintptr_t)1U) 4066c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer prof_malloc_record_object(ptr, usize, cnt); 4076c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer else 4089b0cbf0850b130a9b0a8c58bd10b2926b2083510Jason Evans prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U); 4094d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans} 4104d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 4114d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason EvansJEMALLOC_INLINE void 412665769357cd77b74e00a146f196fff19243b33c4Jason Evansprof_realloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt, 413665769357cd77b74e00a146f196fff19243b33c4Jason Evans size_t old_usize, prof_ctx_t *old_ctx) 4144d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans{ 4154d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans prof_thr_cnt_t *told_cnt; 4164d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 4177372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 4184d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans assert(ptr != NULL || (uintptr_t)cnt <= (uintptr_t)1U); 4194d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 4204d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans if (ptr != NULL) { 421665769357cd77b74e00a146f196fff19243b33c4Jason Evans assert(usize == isalloc(ptr, true)); 4226c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer if (prof_sample_accum_update(usize, true, NULL)) { 4236c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer /* 4246c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer * Don't sample. The usize passed to 4256c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer * PROF_ALLOC_PREP() was larger than what 4266c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer * actually got allocated, so a backtrace was 4276c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer * captured for this allocation, even though 4286c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer * its actual usize was insufficient to cross 4296c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer * the sample threshold. 4306c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer */ 4316c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer cnt = (prof_thr_cnt_t *)(uintptr_t)1U; 4324d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans } 4334d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans } 4344d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 4354d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans if ((uintptr_t)old_ctx > (uintptr_t)1U) { 4364d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans told_cnt = prof_lookup(old_ctx->bt); 4374d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans if (told_cnt == NULL) { 4384d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans /* 4394d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans * It's too late to propagate OOM for this realloc(), 4404d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans * so operate directly on old_cnt->ctx->cnt_merged. 4414d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans */ 4426da5418ded9170b087c35960e0010006430117c1Jason Evans malloc_mutex_lock(old_ctx->lock); 4434d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans old_ctx->cnt_merged.curobjs--; 444665769357cd77b74e00a146f196fff19243b33c4Jason Evans old_ctx->cnt_merged.curbytes -= old_usize; 4456da5418ded9170b087c35960e0010006430117c1Jason Evans malloc_mutex_unlock(old_ctx->lock); 4464d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans told_cnt = (prof_thr_cnt_t *)(uintptr_t)1U; 4474d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans } 4484d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans } else 4494d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans told_cnt = (prof_thr_cnt_t *)(uintptr_t)1U; 4504d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 4514d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans if ((uintptr_t)told_cnt > (uintptr_t)1U) 4524d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans told_cnt->epoch++; 4534d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans if ((uintptr_t)cnt > (uintptr_t)1U) { 4549b0cbf0850b130a9b0a8c58bd10b2926b2083510Jason Evans prof_ctx_set(ptr, cnt->ctx); 4554d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans cnt->epoch++; 4563860eac17023933180ef5dfb5bd24077cda57dfeJason Evans } else if (ptr != NULL) 4579b0cbf0850b130a9b0a8c58bd10b2926b2083510Jason Evans prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U); 4584d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans /*********/ 4594d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans mb_write(); 4604d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans /*********/ 4614d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans if ((uintptr_t)told_cnt > (uintptr_t)1U) { 4624d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans told_cnt->cnts.curobjs--; 463665769357cd77b74e00a146f196fff19243b33c4Jason Evans told_cnt->cnts.curbytes -= old_usize; 4644d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans } 4654d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans if ((uintptr_t)cnt > (uintptr_t)1U) { 4664d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans cnt->cnts.curobjs++; 467665769357cd77b74e00a146f196fff19243b33c4Jason Evans cnt->cnts.curbytes += usize; 4684d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans if (opt_prof_accum) { 4694d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans cnt->cnts.accumobjs++; 470665769357cd77b74e00a146f196fff19243b33c4Jason Evans cnt->cnts.accumbytes += usize; 4714d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans } 4724d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans } 4734d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans /*********/ 4744d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans mb_write(); 4754d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans /*********/ 4764d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans if ((uintptr_t)told_cnt > (uintptr_t)1U) 4774d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans told_cnt->epoch++; 4784d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans if ((uintptr_t)cnt > (uintptr_t)1U) 4794d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans cnt->epoch++; 4804d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans /*********/ 4814d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans mb_write(); /* Not strictly necessary. */ 4824d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans} 4834d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 4844d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason EvansJEMALLOC_INLINE void 485e4f7846f1fd279a039ffa2a41707348187219de4Jason Evansprof_free(const void *ptr, size_t size) 4864d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans{ 4874d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans prof_ctx_t *ctx = prof_ctx_get(ptr); 4884d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 4897372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 4907372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans 4914d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans if ((uintptr_t)ctx > (uintptr_t)1) { 4928b49971d0ce0819af78aa2a278c26ecb298ee134Mike Hommey prof_thr_cnt_t *tcnt; 493122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(size == isalloc(ptr, true)); 4948b49971d0ce0819af78aa2a278c26ecb298ee134Mike Hommey tcnt = prof_lookup(ctx->bt); 4954d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 4964d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans if (tcnt != NULL) { 4974d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans tcnt->epoch++; 4984d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans /*********/ 4994d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans mb_write(); 5004d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans /*********/ 5014d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans tcnt->cnts.curobjs--; 5024d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans tcnt->cnts.curbytes -= size; 5034d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans /*********/ 5044d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans mb_write(); 5054d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans /*********/ 5064d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans tcnt->epoch++; 5074d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans /*********/ 5084d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans mb_write(); 5094d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans /*********/ 5104d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans } else { 5114d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans /* 5124d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans * OOM during free() cannot be propagated, so operate 5134d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans * directly on cnt->ctx->cnt_merged. 5144d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans */ 5156da5418ded9170b087c35960e0010006430117c1Jason Evans malloc_mutex_lock(ctx->lock); 5164d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans ctx->cnt_merged.curobjs--; 5174d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans ctx->cnt_merged.curbytes -= size; 5186da5418ded9170b087c35960e0010006430117c1Jason Evans malloc_mutex_unlock(ctx->lock); 5194d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans } 5204d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans } 5214d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans} 5224d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans#endif 5234d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 5246109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif /* JEMALLOC_H_INLINES */ 5256109fe07a14b7a619365977d9523db9f8b333792Jason Evans/******************************************************************************/ 526