prof.h revision 5b8ed5b7c91939f64f14fc48be84ed20e3f023f4
16109fe07a14b7a619365977d9523db9f8b333792Jason Evans/******************************************************************************/ 26109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_H_TYPES 36109fe07a14b7a619365977d9523db9f8b333792Jason Evans 46109fe07a14b7a619365977d9523db9f8b333792Jason Evanstypedef struct prof_bt_s prof_bt_t; 56109fe07a14b7a619365977d9523db9f8b333792Jason Evanstypedef struct prof_cnt_s prof_cnt_t; 6602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evanstypedef struct prof_tctx_s prof_tctx_t; 7602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evanstypedef struct prof_gctx_s prof_gctx_t; 84d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evanstypedef struct prof_tdata_s prof_tdata_t; 96109fe07a14b7a619365977d9523db9f8b333792Jason Evans 10b9477e782b07afa38c4b1dc0688e053be8a84dd8Jason Evans/* Option defaults. */ 115f60afa01eb2cf7d44024d162a1ecc6cceedcca1Jason Evans#ifdef JEMALLOC_PROF 125f60afa01eb2cf7d44024d162a1ecc6cceedcca1Jason Evans# define PROF_PREFIX_DEFAULT "jeprof" 135f60afa01eb2cf7d44024d162a1ecc6cceedcca1Jason Evans#else 145f60afa01eb2cf7d44024d162a1ecc6cceedcca1Jason Evans# define PROF_PREFIX_DEFAULT "" 155f60afa01eb2cf7d44024d162a1ecc6cceedcca1Jason Evans#endif 160b25fe79aaf8840a5acda7e3160a053d42349872Jason Evans#define LG_PROF_SAMPLE_DEFAULT 19 1737dab02e524c14cffa4bb938242e4cf2695a4a7dJason Evans#define LG_PROF_INTERVAL_DEFAULT -1 180b526ff94da7e59aa947a4d3529b2376794f8b01Jason Evans 196109fe07a14b7a619365977d9523db9f8b333792Jason Evans/* 205389146191b279ca3b90028357dd6ad66b283defJason Evans * Hard limit on stack backtrace depth. The version of prof_backtrace() that 215389146191b279ca3b90028357dd6ad66b283defJason Evans * is based on __builtin_return_address() necessarily has a hard-coded number 225389146191b279ca3b90028357dd6ad66b283defJason Evans * of backtrace frame handlers, and should be kept in sync with this setting. 236109fe07a14b7a619365977d9523db9f8b333792Jason Evans */ 245389146191b279ca3b90028357dd6ad66b283defJason Evans#define PROF_BT_MAX 128 255389146191b279ca3b90028357dd6ad66b283defJason Evans 266109fe07a14b7a619365977d9523db9f8b333792Jason Evans/* Initial hash table size. */ 27cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans#define PROF_CKH_MINITEMS 64 286109fe07a14b7a619365977d9523db9f8b333792Jason Evans 296109fe07a14b7a619365977d9523db9f8b333792Jason Evans/* Size of memory buffer to use when writing dump files. */ 30cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans#define PROF_DUMP_BUFSIZE 65536 31cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans 32cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans/* Size of stack-allocated buffer used by prof_printf(). */ 33cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans#define PROF_PRINTF_BUFSIZE 128 346109fe07a14b7a619365977d9523db9f8b333792Jason Evans 356da5418ded9170b087c35960e0010006430117c1Jason Evans/* 36602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * Number of mutexes shared among all gctx's. No space is allocated for these 376da5418ded9170b087c35960e0010006430117c1Jason Evans * unless profiling is enabled, so it's okay to over-provision. 386da5418ded9170b087c35960e0010006430117c1Jason Evans */ 396da5418ded9170b087c35960e0010006430117c1Jason Evans#define PROF_NCTX_LOCKS 1024 406da5418ded9170b087c35960e0010006430117c1Jason Evans 410050a0f7e6ea5a33c9aed769e2652afe20714194Jason Evans/* 42602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * Number of mutexes shared among all tdata's. No space is allocated for these 43602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * unless profiling is enabled, so it's okay to over-provision. 44602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans */ 45602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans#define PROF_NTDATA_LOCKS 256 46602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 47602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans/* 480050a0f7e6ea5a33c9aed769e2652afe20714194Jason Evans * prof_tdata pointers close to NULL are used to encode state information that 490050a0f7e6ea5a33c9aed769e2652afe20714194Jason Evans * is used for cleaning up during thread shutdown. 500050a0f7e6ea5a33c9aed769e2652afe20714194Jason Evans */ 510050a0f7e6ea5a33c9aed769e2652afe20714194Jason Evans#define PROF_TDATA_STATE_REINCARNATED ((prof_tdata_t *)(uintptr_t)1) 520050a0f7e6ea5a33c9aed769e2652afe20714194Jason Evans#define PROF_TDATA_STATE_PURGATORY ((prof_tdata_t *)(uintptr_t)2) 530050a0f7e6ea5a33c9aed769e2652afe20714194Jason Evans#define PROF_TDATA_STATE_MAX PROF_TDATA_STATE_PURGATORY 540050a0f7e6ea5a33c9aed769e2652afe20714194Jason Evans 556109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif /* JEMALLOC_H_TYPES */ 566109fe07a14b7a619365977d9523db9f8b333792Jason Evans/******************************************************************************/ 576109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_H_STRUCTS 586109fe07a14b7a619365977d9523db9f8b333792Jason Evans 596109fe07a14b7a619365977d9523db9f8b333792Jason Evansstruct prof_bt_s { 606109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* Backtrace, stored as len program counters. */ 614d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans void **vec; 624d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans unsigned len; 636109fe07a14b7a619365977d9523db9f8b333792Jason Evans}; 646109fe07a14b7a619365977d9523db9f8b333792Jason Evans 65b27805b36309681da1936eb33044584547552340Jason Evans#ifdef JEMALLOC_PROF_LIBGCC 66b27805b36309681da1936eb33044584547552340Jason Evans/* Data structure passed to libgcc _Unwind_Backtrace() callback functions. */ 67b27805b36309681da1936eb33044584547552340Jason Evanstypedef struct { 68a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans prof_bt_t *bt; 69a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans unsigned max; 70b27805b36309681da1936eb33044584547552340Jason Evans} prof_unwind_data_t; 71b27805b36309681da1936eb33044584547552340Jason Evans#endif 72b27805b36309681da1936eb33044584547552340Jason Evans 736109fe07a14b7a619365977d9523db9f8b333792Jason Evansstruct prof_cnt_s { 74602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans /* Profiling counters. */ 75602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans uint64_t curobjs; 76602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans uint64_t curbytes; 776109fe07a14b7a619365977d9523db9f8b333792Jason Evans uint64_t accumobjs; 786109fe07a14b7a619365977d9523db9f8b333792Jason Evans uint64_t accumbytes; 796109fe07a14b7a619365977d9523db9f8b333792Jason Evans}; 806109fe07a14b7a619365977d9523db9f8b333792Jason Evans 81602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evanstypedef enum { 826ef80d68f092caf3b3802a73b8d716057b41864cJason Evans prof_tctx_state_initializing, 83602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tctx_state_nominal, 84602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tctx_state_dumping, 85602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tctx_state_purgatory /* Dumper must finish destroying. */ 86602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans} prof_tctx_state_t; 873a81cbd2d4f2d8c052f11f4b0b73ee5c84a33d4fJason Evans 88602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansstruct prof_tctx_s { 89602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans /* Thread data for thread that performed the allocation. */ 90602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tdata_t *tdata; 91a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans 9244c97b712ef1669a4c75ea97e8d47c0535e9ec71Jason Evans /* 9344c97b712ef1669a4c75ea97e8d47c0535e9ec71Jason Evans * Copy of tdata->thr_uid, necessary because tdata may be defunct during 9444c97b712ef1669a4c75ea97e8d47c0535e9ec71Jason Evans * teardown. 9544c97b712ef1669a4c75ea97e8d47c0535e9ec71Jason Evans */ 9644c97b712ef1669a4c75ea97e8d47c0535e9ec71Jason Evans uint64_t thr_uid; 9744c97b712ef1669a4c75ea97e8d47c0535e9ec71Jason Evans 98602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans /* Profiling counters, protected by tdata->lock. */ 99602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_cnt_t cnts; 100602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 101602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans /* Associated global context. */ 102602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_gctx_t *gctx; 103602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 104602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans /* Linkage into gctx's tctxs. */ 105602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans rb_node(prof_tctx_t) tctx_link; 106602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 1076e73dc194ee9682d3eacaf725a989f04629718f7Jason Evans /* 1086e73dc194ee9682d3eacaf725a989f04629718f7Jason Evans * True during prof_alloc_prep()..prof_malloc_sample_object(), prevents 1096e73dc194ee9682d3eacaf725a989f04629718f7Jason Evans * sample vs destroy race. 1106e73dc194ee9682d3eacaf725a989f04629718f7Jason Evans */ 1116e73dc194ee9682d3eacaf725a989f04629718f7Jason Evans bool prepared; 1126e73dc194ee9682d3eacaf725a989f04629718f7Jason Evans 113602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans /* Current dump-related state, protected by gctx->lock. */ 114602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tctx_state_t state; 1156109fe07a14b7a619365977d9523db9f8b333792Jason Evans 1166109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* 117602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * Copy of cnts snapshotted during early dump phase, protected by 118602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * dump_mtx. 1196109fe07a14b7a619365977d9523db9f8b333792Jason Evans */ 120602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_cnt_t dump_cnts; 1216109fe07a14b7a619365977d9523db9f8b333792Jason Evans}; 122602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evanstypedef rb_tree(prof_tctx_t) prof_tctx_tree_t; 1236109fe07a14b7a619365977d9523db9f8b333792Jason Evans 124602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansstruct prof_gctx_s { 125602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans /* Protects nlimbo, cnt_summed, and tctxs. */ 1266da5418ded9170b087c35960e0010006430117c1Jason Evans malloc_mutex_t *lock; 1276109fe07a14b7a619365977d9523db9f8b333792Jason Evans 12852386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans /* 129602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * Number of threads that currently cause this gctx to be in a state of 13052386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans * limbo due to one of: 131602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * - Initializing this gctx. 132602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * - Initializing per thread counters associated with this gctx. 133602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * - Preparing to destroy this gctx. 134602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * - Dumping a heap profile that includes this gctx. 13552386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans * nlimbo must be 1 (single destroyer) in order to safely destroy the 136602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * gctx. 13752386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans */ 13852386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans unsigned nlimbo; 13952386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans 1406109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* 1413a81cbd2d4f2d8c052f11f4b0b73ee5c84a33d4fJason Evans * Tree of profile counters, one for each thread that has allocated in 1426109fe07a14b7a619365977d9523db9f8b333792Jason Evans * this context. 1436109fe07a14b7a619365977d9523db9f8b333792Jason Evans */ 144602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tctx_tree_t tctxs; 1454f37ef693e3d5903ce07dc0b61c0da320b35e3d9Jason Evans 1463a81cbd2d4f2d8c052f11f4b0b73ee5c84a33d4fJason Evans /* Linkage for tree of contexts to be dumped. */ 147602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans rb_node(prof_gctx_t) dump_link; 148602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 149602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans /* Temporary storage for summation during dump. */ 150602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_cnt_t cnt_summed; 151ab532e97991d190e9368781cf308c60c2319b933Jason Evans 152ab532e97991d190e9368781cf308c60c2319b933Jason Evans /* Associated backtrace. */ 153ab532e97991d190e9368781cf308c60c2319b933Jason Evans prof_bt_t bt; 154ab532e97991d190e9368781cf308c60c2319b933Jason Evans 155ab532e97991d190e9368781cf308c60c2319b933Jason Evans /* Backtrace vector, variable size, referred to by bt. */ 156ab532e97991d190e9368781cf308c60c2319b933Jason Evans void *vec[1]; 1576109fe07a14b7a619365977d9523db9f8b333792Jason Evans}; 158602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evanstypedef rb_tree(prof_gctx_t) prof_gctx_tree_t; 159602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 1604d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evansstruct prof_tdata_s { 161602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans malloc_mutex_t *lock; 162602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 163602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans /* Monotonically increasing unique thread identifier. */ 164602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans uint64_t thr_uid; 165602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 16620c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evans /* 16720c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evans * Monotonically increasing discriminator among tdata structures 16820c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evans * associated with the same thr_uid. 16920c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evans */ 17020c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evans uint64_t thr_discrim; 17120c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evans 172602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans /* Included in heap profile dumps if non-NULL. */ 173602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans char *thread_name; 174602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 17520c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evans bool attached; 17620c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evans bool expired; 177602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 178602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans rb_node(prof_tdata_t) tdata_link; 179602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 1804d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans /* 181602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * Hash of (prof_bt_t *)-->(prof_tctx_t *). Each thread tracks 182b41ccdb125b312d4522da1a80091a0137773c964Jason Evans * backtraces for which it has non-zero allocation/deallocation counters 183602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * associated with thread-specific prof_tctx_t objects. Other threads 184602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * may write to prof_tctx_t contents when freeing associated objects. 1854d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans */ 186602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans ckh_t bt2tctx; 187a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans 1884d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans /* Sampling state. */ 18984f7cdb0c588322dfd50a26497fc1cb54b792018Jason Evans uint64_t prng_state; 1906c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer uint64_t bytes_until_sample; 19152386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans 19252386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans /* State used to avoid dumping while operating on prof internals. */ 19352386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans bool enq; 19452386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans bool enq_idump; 19552386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans bool enq_gdump; 196b41ccdb125b312d4522da1a80091a0137773c964Jason Evans 197602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans /* 198602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * Set to true during an early dump phase for tdata's which are 199602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * currently being dumped. New threads' tdata's have this initialized 200602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * to false so that they aren't accidentally included in later dump 201602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * phases. 202602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans */ 203602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans bool dumping; 204602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 205602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans /* 206602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * True if profiling is active for this tdata's thread 207602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * (thread.prof.active mallctl). 208602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans */ 209602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans bool active; 210602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 211602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans /* Temporary storage for summation during dump. */ 212602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_cnt_t cnt_summed; 213602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 214b41ccdb125b312d4522da1a80091a0137773c964Jason Evans /* Backtrace vector, used for calls to prof_backtrace(). */ 215b41ccdb125b312d4522da1a80091a0137773c964Jason Evans void *vec[PROF_BT_MAX]; 216a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans}; 217602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evanstypedef rb_tree(prof_tdata_t) prof_tdata_tree_t; 218a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans 2196109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif /* JEMALLOC_H_STRUCTS */ 2206109fe07a14b7a619365977d9523db9f8b333792Jason Evans/******************************************************************************/ 2216109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_H_EXTERNS 2226109fe07a14b7a619365977d9523db9f8b333792Jason Evans 2236109fe07a14b7a619365977d9523db9f8b333792Jason Evansextern bool opt_prof; 224f18c98200145de70779a1b3286e7829b0268231eJason Evansextern bool opt_prof_active; 225fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evansextern bool opt_prof_thread_active_init; 226a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evansextern size_t opt_lg_prof_sample; /* Mean bytes between samples. */ 227a02fc08ec9dd8479a6430155b6a433da09f6ff10Jason Evansextern ssize_t opt_lg_prof_interval; /* lg(prof_interval). */ 228e73397062ac3ab28a9d377591b63ed19fd154ccaJason Evansextern bool opt_prof_gdump; /* High-water memory dumping. */ 2290b25fe79aaf8840a5acda7e3160a053d42349872Jason Evansextern bool opt_prof_final; /* Final profile dumping. */ 230a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evansextern bool opt_prof_leak; /* Dump leak summary at exit. */ 231a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evansextern bool opt_prof_accum; /* Report cumulative bytes. */ 232eefdd02e70ec1b9cf11920fcff585835dcbd766bJason Evansextern char opt_prof_prefix[ 233eefdd02e70ec1b9cf11920fcff585835dcbd766bJason Evans /* Minimize memory bloat for non-prof builds. */ 234eefdd02e70ec1b9cf11920fcff585835dcbd766bJason Evans#ifdef JEMALLOC_PROF 235eefdd02e70ec1b9cf11920fcff585835dcbd766bJason Evans PATH_MAX + 236eefdd02e70ec1b9cf11920fcff585835dcbd766bJason Evans#endif 237eefdd02e70ec1b9cf11920fcff585835dcbd766bJason Evans 1]; 2386109fe07a14b7a619365977d9523db9f8b333792Jason Evans 239fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans/* Accessed via prof_active_[gs]et{_unlocked,}(). */ 240fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evansextern bool prof_active; 241fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans 2425b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evans/* Accessed via prof_gdump_[gs]et{_unlocked,}(). */ 2435b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evansextern bool prof_gdump_val; 2445b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evans 245d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans/* 246d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans * Profile dump interval, measured in bytes allocated. Each arena triggers a 247d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans * profile dump when it reaches this threshold. The effect is that the 248d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans * interval between profile dumps averages prof_interval, though the actual 249d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans * interval between dumps will tend to be sporadic, and the interval will be a 250d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans * maximum of approximately (prof_interval * narenas). 251d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans */ 252d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evansextern uint64_t prof_interval; 253d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans 254602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans/* 255602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * Initialized as opt_lg_prof_sample, and potentially modified during profiling 256602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * resets. 257602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans */ 258602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansextern size_t lg_prof_sample; 259602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 2605460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansvoid prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated); 261602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansvoid prof_malloc_sample_object(const void *ptr, size_t usize, 262602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tctx_t *tctx); 2635460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansvoid prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx); 2644d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evansvoid bt_init(prof_bt_t *bt, void **vec); 2656f001059aa33d77a3cb7799002044faf8dd08fc0Jason Evansvoid prof_backtrace(prof_bt_t *bt); 2665460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansprof_tctx_t *prof_lookup(tsd_t *tsd, prof_bt_t *bt); 267772163b4f3d8e9a12343e9215f6b070068507604Jason Evans#ifdef JEMALLOC_JET 26820c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evanssize_t prof_tdata_count(void); 269772163b4f3d8e9a12343e9215f6b070068507604Jason Evanssize_t prof_bt_count(void); 27020c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evansconst prof_cnt_t *prof_cnt_all(void); 271772163b4f3d8e9a12343e9215f6b070068507604Jason Evanstypedef int (prof_dump_open_t)(bool, const char *); 272772163b4f3d8e9a12343e9215f6b070068507604Jason Evansextern prof_dump_open_t *prof_dump_open; 27320c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evanstypedef bool (prof_dump_header_t)(bool, const prof_cnt_t *); 27420c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evansextern prof_dump_header_t *prof_dump_header; 275772163b4f3d8e9a12343e9215f6b070068507604Jason Evans#endif 2766109fe07a14b7a619365977d9523db9f8b333792Jason Evansvoid prof_idump(void); 27722ca855e8fbf4c592f5e46aaec381b68de60c71aJason Evansbool prof_mdump(const char *filename); 278e73397062ac3ab28a9d377591b63ed19fd154ccaJason Evansvoid prof_gdump(void); 2795460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansprof_tdata_t *prof_tdata_init(tsd_t *tsd); 2805460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansprof_tdata_t *prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata); 2815460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansvoid prof_reset(tsd_t *tsd, size_t lg_sample); 2825460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansvoid prof_tdata_cleanup(tsd_t *tsd); 283602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansconst char *prof_thread_name_get(void); 284fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evansbool prof_active_get(void); 285fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evansbool prof_active_set(bool active); 286fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evansint prof_thread_name_set(tsd_t *tsd, const char *thread_name); 287602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansbool prof_thread_active_get(void); 288602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansbool prof_thread_active_set(bool active); 289fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evansbool prof_thread_active_init_get(void); 290fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evansbool prof_thread_active_init_set(bool active_init); 2915b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evansbool prof_gdump_get(void); 2925b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evansbool prof_gdump_set(bool active); 2936109fe07a14b7a619365977d9523db9f8b333792Jason Evansvoid prof_boot0(void); 294e73397062ac3ab28a9d377591b63ed19fd154ccaJason Evansvoid prof_boot1(void); 295e73397062ac3ab28a9d377591b63ed19fd154ccaJason Evansbool prof_boot2(void); 29620f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evansvoid prof_prefork(void); 29720f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evansvoid prof_postfork_parent(void); 29820f1fc95adb35ea63dc61f47f2b0ffbd37d39f32Jason Evansvoid prof_postfork_child(void); 299602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansvoid prof_sample_threshold_update(prof_tdata_t *tdata); 3006109fe07a14b7a619365977d9523db9f8b333792Jason Evans 3016109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif /* JEMALLOC_H_EXTERNS */ 3026109fe07a14b7a619365977d9523db9f8b333792Jason Evans/******************************************************************************/ 3036109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_H_INLINES 3046109fe07a14b7a619365977d9523db9f8b333792Jason Evans 3054d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans#ifndef JEMALLOC_ENABLE_INLINE 306fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evansbool prof_active_get_unlocked(void); 3075b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evansbool prof_gdump_get_unlocked(void); 3085460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansprof_tdata_t *prof_tdata_get(tsd_t *tsd, bool create); 3095460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansbool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool commit, 310602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tdata_t **tdata_out); 3115460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansprof_tctx_t *prof_alloc_prep(tsd_t *tsd, size_t usize, bool update); 312602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansprof_tctx_t *prof_tctx_get(const void *ptr); 313602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansvoid prof_tctx_set(const void *ptr, prof_tctx_t *tctx); 314602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansvoid prof_malloc_sample_object(const void *ptr, size_t usize, 315602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tctx_t *tctx); 316602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansvoid prof_malloc(const void *ptr, size_t usize, prof_tctx_t *tctx); 3175460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansvoid prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, 3185460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans prof_tctx_t *tctx, bool updated, size_t old_usize, prof_tctx_t *old_tctx); 3195460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansvoid prof_free(tsd_t *tsd, const void *ptr, size_t usize); 3204d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans#endif 3214d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 3224d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_)) 32334e85b4182d5ae029b558aae3da25fff7c3efe12Jason EvansJEMALLOC_ALWAYS_INLINE bool 324fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evansprof_active_get_unlocked(void) 325fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans{ 326fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans 327fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans /* 328fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans * Even if opt_prof is true, sampling can be temporarily disabled by 329fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans * setting prof_active to false. No locking is used when reading 330fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans * prof_active in the fast path, so there are no guarantees regarding 331fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans * how long it will take for all threads to notice state changes. 332fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans */ 333fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans return (prof_active); 334fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans} 335fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans 3365b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason EvansJEMALLOC_ALWAYS_INLINE bool 3375b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evansprof_gdump_get_unlocked(void) 3385b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evans{ 3395b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evans 3405b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evans /* 3415b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evans * No locking is used when reading prof_gdump_val in the fast path, so 3425b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evans * there are no guarantees regarding how long it will take for all 3435b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evans * threads to notice state changes. 3445b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evans */ 3455b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evans return (prof_gdump_val); 3465b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evans} 3475b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evans 34834e85b4182d5ae029b558aae3da25fff7c3efe12Jason EvansJEMALLOC_ALWAYS_INLINE prof_tdata_t * 3495460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansprof_tdata_get(tsd_t *tsd, bool create) 35052386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans{ 351602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tdata_t *tdata; 35252386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans 35352386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans cassert(config_prof); 35452386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans 3555460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans tdata = tsd_prof_tdata_get(tsd); 356602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans if (create) { 3575460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans if (unlikely(tdata == NULL)) { 358029d44cf8b22aa7b749747bfd585887fb59e0030Jason Evans if (tsd_nominal(tsd)) { 359029d44cf8b22aa7b749747bfd585887fb59e0030Jason Evans tdata = prof_tdata_init(tsd); 360029d44cf8b22aa7b749747bfd585887fb59e0030Jason Evans tsd_prof_tdata_set(tsd, tdata); 361029d44cf8b22aa7b749747bfd585887fb59e0030Jason Evans } 36220c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evans } else if (unlikely(tdata->expired)) { 3635460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans tdata = prof_tdata_reinit(tsd, tdata); 3645460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans tsd_prof_tdata_set(tsd, tdata); 3655460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans } 36620c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evans assert(tdata == NULL || tdata->attached); 367602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans } 36852386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans 369602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans return (tdata); 37052386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans} 37152386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans 37234e85b4182d5ae029b558aae3da25fff7c3efe12Jason EvansJEMALLOC_ALWAYS_INLINE prof_tctx_t * 373602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansprof_tctx_get(const void *ptr) 3744d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans{ 375602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tctx_t *ret; 3764d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans arena_chunk_t *chunk; 3774d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 3787372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 3794d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans assert(ptr != NULL); 3804d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 3814d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 382cfc5706f6977a48f3b82d69cd68aa1cf8802fb8dJason Evans if (likely(chunk != ptr)) 383602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans ret = arena_prof_tctx_get(ptr); 384cfc5706f6977a48f3b82d69cd68aa1cf8802fb8dJason Evans else 385602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans ret = huge_prof_tctx_get(ptr); 3864d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 3874d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans return (ret); 3884d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans} 3894d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 39034e85b4182d5ae029b558aae3da25fff7c3efe12Jason EvansJEMALLOC_ALWAYS_INLINE void 391602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansprof_tctx_set(const void *ptr, prof_tctx_t *tctx) 3924d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans{ 3934d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans arena_chunk_t *chunk; 3944d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 3957372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 3964d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans assert(ptr != NULL); 3974d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 3984d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 399cfc5706f6977a48f3b82d69cd68aa1cf8802fb8dJason Evans if (likely(chunk != ptr)) 400602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans arena_prof_tctx_set(ptr, tctx); 401cfc5706f6977a48f3b82d69cd68aa1cf8802fb8dJason Evans else 402602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans huge_prof_tctx_set(ptr, tctx); 4034d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans} 4044d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 40534e85b4182d5ae029b558aae3da25fff7c3efe12Jason EvansJEMALLOC_ALWAYS_INLINE bool 4065460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansprof_sample_accum_update(tsd_t *tsd, size_t usize, bool update, 4075460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans prof_tdata_t **tdata_out) 4084d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans{ 409602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tdata_t *tdata; 4104d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 4117372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 4124d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 4135460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans tdata = prof_tdata_get(tsd, true); 414602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) 415602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans tdata = NULL; 4166c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer 417602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans if (tdata_out != NULL) 418602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans *tdata_out = tdata; 4196c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer 420602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans if (tdata == NULL) 421f27899402914065a6c1484ea8d81a2c8b70aa659Jason Evans return (true); 4224d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 423602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans if (tdata->bytes_until_sample >= usize) { 4246e73dc194ee9682d3eacaf725a989f04629718f7Jason Evans if (update) 425602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans tdata->bytes_until_sample -= usize; 4266c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer return (true); 4276c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer } else { 428ff7450727f64180367f430b1b747f9e682e26df4Jason Evans /* Compute new sample threshold. */ 4296e73dc194ee9682d3eacaf725a989f04629718f7Jason Evans if (update) 430602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_sample_threshold_update(tdata); 431551ebc43647521bdd0bc78558b106762b3388928Jason Evans return (!tdata->active); 4324d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans } 4334d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans} 4344d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 43534e85b4182d5ae029b558aae3da25fff7c3efe12Jason EvansJEMALLOC_ALWAYS_INLINE prof_tctx_t * 4365460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansprof_alloc_prep(tsd_t *tsd, size_t usize, bool update) 437602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans{ 438602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tctx_t *ret; 439602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tdata_t *tdata; 440602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_bt_t bt; 441602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 442602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans assert(usize == s2u(usize)); 443602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 444fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans if (!prof_active_get_unlocked() || likely(prof_sample_accum_update(tsd, 445fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans usize, update, &tdata))) 446602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans ret = (prof_tctx_t *)(uintptr_t)1U; 447602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans else { 448602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans bt_init(&bt, tdata->vec); 449602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_backtrace(&bt); 4505460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans ret = prof_lookup(tsd, &bt); 4516c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer } 452602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 453602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans return (ret); 4546c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer} 4556c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer 45634e85b4182d5ae029b558aae3da25fff7c3efe12Jason EvansJEMALLOC_ALWAYS_INLINE void 457602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansprof_malloc(const void *ptr, size_t usize, prof_tctx_t *tctx) 4584d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans{ 4594d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 4607372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 4614d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans assert(ptr != NULL); 462665769357cd77b74e00a146f196fff19243b33c4Jason Evans assert(usize == isalloc(ptr, true)); 4634d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 4649c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) 465602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_malloc_sample_object(ptr, usize, tctx); 4666c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer else 467602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tctx_set(ptr, (prof_tctx_t *)(uintptr_t)1U); 4684d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans} 4694d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 47034e85b4182d5ae029b558aae3da25fff7c3efe12Jason EvansJEMALLOC_ALWAYS_INLINE void 4715460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansprof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx, 4725460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans bool updated, size_t old_usize, prof_tctx_t *old_tctx) 4734d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans{ 4744d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 4757372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 476602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans assert(ptr != NULL || (uintptr_t)tctx <= (uintptr_t)1U); 4774d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 4786e73dc194ee9682d3eacaf725a989f04629718f7Jason Evans if (!updated && ptr != NULL) { 479665769357cd77b74e00a146f196fff19243b33c4Jason Evans assert(usize == isalloc(ptr, true)); 4805460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans if (prof_sample_accum_update(tsd, usize, true, NULL)) { 4816c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer /* 482602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * Don't sample. The usize passed to PROF_ALLOC_PREP() 483602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * was larger than what actually got allocated, so a 484602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * backtrace was captured for this allocation, even 485602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * though its actual usize was insufficient to cross the 486602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * sample threshold. 4876c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer */ 488602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans tctx = (prof_tctx_t *)(uintptr_t)1U; 4894d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans } 4904d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans } 4914d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 4929c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (unlikely((uintptr_t)old_tctx > (uintptr_t)1U)) 4935460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans prof_free_sampled_object(tsd, old_usize, old_tctx); 4949c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) 495602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_malloc_sample_object(ptr, usize, tctx); 496602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans else 497602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tctx_set(ptr, (prof_tctx_t *)(uintptr_t)1U); 4984d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans} 4994d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 50034e85b4182d5ae029b558aae3da25fff7c3efe12Jason EvansJEMALLOC_ALWAYS_INLINE void 5015460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansprof_free(tsd_t *tsd, const void *ptr, size_t usize) 5024d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans{ 503602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tctx_t *tctx = prof_tctx_get(ptr); 5044d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 5057372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 506602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans assert(usize == isalloc(ptr, true)); 5077372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans 5089c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) 5095460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans prof_free_sampled_object(tsd, usize, tctx); 5104d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans} 5114d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans#endif 5124d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 5136109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif /* JEMALLOC_H_INLINES */ 5146109fe07a14b7a619365977d9523db9f8b333792Jason Evans/******************************************************************************/ 515