16109fe07a14b7a619365977d9523db9f8b333792Jason Evans/******************************************************************************/ 26109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_H_TYPES 36109fe07a14b7a619365977d9523db9f8b333792Jason Evans 46109fe07a14b7a619365977d9523db9f8b333792Jason Evanstypedef struct prof_bt_s prof_bt_t; 56109fe07a14b7a619365977d9523db9f8b333792Jason Evanstypedef struct prof_cnt_s prof_cnt_t; 6602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evanstypedef struct prof_tctx_s prof_tctx_t; 7602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evanstypedef struct prof_gctx_s prof_gctx_t; 84d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evanstypedef struct prof_tdata_s prof_tdata_t; 96109fe07a14b7a619365977d9523db9f8b333792Jason Evans 10b9477e782b07afa38c4b1dc0688e053be8a84dd8Jason Evans/* Option defaults. */ 115f60afa01eb2cf7d44024d162a1ecc6cceedcca1Jason Evans#ifdef JEMALLOC_PROF 125f60afa01eb2cf7d44024d162a1ecc6cceedcca1Jason Evans# define PROF_PREFIX_DEFAULT "jeprof" 135f60afa01eb2cf7d44024d162a1ecc6cceedcca1Jason Evans#else 145f60afa01eb2cf7d44024d162a1ecc6cceedcca1Jason Evans# define PROF_PREFIX_DEFAULT "" 155f60afa01eb2cf7d44024d162a1ecc6cceedcca1Jason Evans#endif 160b25fe79aaf8840a5acda7e3160a053d42349872Jason Evans#define LG_PROF_SAMPLE_DEFAULT 19 1737dab02e524c14cffa4bb938242e4cf2695a4a7dJason Evans#define LG_PROF_INTERVAL_DEFAULT -1 180b526ff94da7e59aa947a4d3529b2376794f8b01Jason Evans 196109fe07a14b7a619365977d9523db9f8b333792Jason Evans/* 205389146191b279ca3b90028357dd6ad66b283defJason Evans * Hard limit on stack backtrace depth. The version of prof_backtrace() that 215389146191b279ca3b90028357dd6ad66b283defJason Evans * is based on __builtin_return_address() necessarily has a hard-coded number 225389146191b279ca3b90028357dd6ad66b283defJason Evans * of backtrace frame handlers, and should be kept in sync with this setting. 236109fe07a14b7a619365977d9523db9f8b333792Jason Evans */ 245389146191b279ca3b90028357dd6ad66b283defJason Evans#define PROF_BT_MAX 128 255389146191b279ca3b90028357dd6ad66b283defJason Evans 266109fe07a14b7a619365977d9523db9f8b333792Jason Evans/* Initial hash table size. */ 27cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans#define PROF_CKH_MINITEMS 64 286109fe07a14b7a619365977d9523db9f8b333792Jason Evans 296109fe07a14b7a619365977d9523db9f8b333792Jason Evans/* Size of memory buffer to use when writing dump files. */ 30cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans#define PROF_DUMP_BUFSIZE 65536 31cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans 32cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans/* Size of stack-allocated buffer used by prof_printf(). */ 33cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans#define PROF_PRINTF_BUFSIZE 128 346109fe07a14b7a619365977d9523db9f8b333792Jason Evans 356da5418ded9170b087c35960e0010006430117c1Jason Evans/* 36602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * Number of mutexes shared among all gctx's. No space is allocated for these 376da5418ded9170b087c35960e0010006430117c1Jason Evans * unless profiling is enabled, so it's okay to over-provision. 386da5418ded9170b087c35960e0010006430117c1Jason Evans */ 396da5418ded9170b087c35960e0010006430117c1Jason Evans#define PROF_NCTX_LOCKS 1024 406da5418ded9170b087c35960e0010006430117c1Jason Evans 410050a0f7e6ea5a33c9aed769e2652afe20714194Jason Evans/* 42602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * Number of mutexes shared among all tdata's. No space is allocated for these 43602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * unless profiling is enabled, so it's okay to over-provision. 44602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans */ 45602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans#define PROF_NTDATA_LOCKS 256 46602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 47602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans/* 480050a0f7e6ea5a33c9aed769e2652afe20714194Jason Evans * prof_tdata pointers close to NULL are used to encode state information that 490050a0f7e6ea5a33c9aed769e2652afe20714194Jason Evans * is used for cleaning up during thread shutdown. 500050a0f7e6ea5a33c9aed769e2652afe20714194Jason Evans */ 510050a0f7e6ea5a33c9aed769e2652afe20714194Jason Evans#define PROF_TDATA_STATE_REINCARNATED ((prof_tdata_t *)(uintptr_t)1) 520050a0f7e6ea5a33c9aed769e2652afe20714194Jason Evans#define PROF_TDATA_STATE_PURGATORY ((prof_tdata_t *)(uintptr_t)2) 530050a0f7e6ea5a33c9aed769e2652afe20714194Jason Evans#define PROF_TDATA_STATE_MAX PROF_TDATA_STATE_PURGATORY 540050a0f7e6ea5a33c9aed769e2652afe20714194Jason Evans 556109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif /* JEMALLOC_H_TYPES */ 566109fe07a14b7a619365977d9523db9f8b333792Jason Evans/******************************************************************************/ 576109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_H_STRUCTS 586109fe07a14b7a619365977d9523db9f8b333792Jason Evans 596109fe07a14b7a619365977d9523db9f8b333792Jason Evansstruct prof_bt_s { 606109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* Backtrace, stored as len program counters. */ 614d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans void **vec; 624d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans unsigned len; 636109fe07a14b7a619365977d9523db9f8b333792Jason Evans}; 646109fe07a14b7a619365977d9523db9f8b333792Jason Evans 65b27805b36309681da1936eb33044584547552340Jason Evans#ifdef JEMALLOC_PROF_LIBGCC 66b27805b36309681da1936eb33044584547552340Jason Evans/* Data structure passed to libgcc _Unwind_Backtrace() callback functions. */ 67b27805b36309681da1936eb33044584547552340Jason Evanstypedef struct { 68a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans prof_bt_t *bt; 69a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans unsigned max; 70b27805b36309681da1936eb33044584547552340Jason Evans} prof_unwind_data_t; 71b27805b36309681da1936eb33044584547552340Jason Evans#endif 72b27805b36309681da1936eb33044584547552340Jason Evans 736109fe07a14b7a619365977d9523db9f8b333792Jason Evansstruct prof_cnt_s { 74602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans /* Profiling counters. */ 75602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans uint64_t curobjs; 76602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans uint64_t curbytes; 776109fe07a14b7a619365977d9523db9f8b333792Jason Evans uint64_t accumobjs; 786109fe07a14b7a619365977d9523db9f8b333792Jason Evans uint64_t accumbytes; 796109fe07a14b7a619365977d9523db9f8b333792Jason Evans}; 806109fe07a14b7a619365977d9523db9f8b333792Jason Evans 81602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evanstypedef enum { 826ef80d68f092caf3b3802a73b8d716057b41864cJason Evans prof_tctx_state_initializing, 83602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tctx_state_nominal, 84602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tctx_state_dumping, 85602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tctx_state_purgatory /* Dumper must finish destroying. */ 86602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans} prof_tctx_state_t; 873a81cbd2d4f2d8c052f11f4b0b73ee5c84a33d4fJason Evans 88602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansstruct prof_tctx_s { 89602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans /* Thread data for thread that performed the allocation. */ 90602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tdata_t *tdata; 91a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans 9244c97b712ef1669a4c75ea97e8d47c0535e9ec71Jason Evans /* 93a00b10735a80f7070714b278c8acdad4473bea69Jason Evans * Copy of tdata->thr_{uid,discrim}, necessary because tdata may be 94a00b10735a80f7070714b278c8acdad4473bea69Jason Evans * defunct during teardown. 9544c97b712ef1669a4c75ea97e8d47c0535e9ec71Jason Evans */ 9644c97b712ef1669a4c75ea97e8d47c0535e9ec71Jason Evans uint64_t thr_uid; 97a00b10735a80f7070714b278c8acdad4473bea69Jason Evans uint64_t thr_discrim; 9844c97b712ef1669a4c75ea97e8d47c0535e9ec71Jason Evans 99602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans /* Profiling counters, protected by tdata->lock. */ 100602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_cnt_t cnts; 101602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 102602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans /* Associated global context. */ 103602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_gctx_t *gctx; 104602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 10504211e226628c41da4b3804ba411b5dd4b3a02abJason Evans /* 10604211e226628c41da4b3804ba411b5dd4b3a02abJason Evans * UID that distinguishes multiple tctx's created by the same thread, 10704211e226628c41da4b3804ba411b5dd4b3a02abJason Evans * but coexisting in gctx->tctxs. There are two ways that such 10804211e226628c41da4b3804ba411b5dd4b3a02abJason Evans * coexistence can occur: 10904211e226628c41da4b3804ba411b5dd4b3a02abJason Evans * - A dumper thread can cause a tctx to be retained in the purgatory 11004211e226628c41da4b3804ba411b5dd4b3a02abJason Evans * state. 11104211e226628c41da4b3804ba411b5dd4b3a02abJason Evans * - Although a single "producer" thread must create all tctx's which 11204211e226628c41da4b3804ba411b5dd4b3a02abJason Evans * share the same thr_uid, multiple "consumers" can each concurrently 11304211e226628c41da4b3804ba411b5dd4b3a02abJason Evans * execute portions of prof_tctx_destroy(). prof_tctx_destroy() only 11404211e226628c41da4b3804ba411b5dd4b3a02abJason Evans * gets called once each time cnts.cur{objs,bytes} drop to 0, but this 11504211e226628c41da4b3804ba411b5dd4b3a02abJason Evans * threshold can be hit again before the first consumer finishes 11604211e226628c41da4b3804ba411b5dd4b3a02abJason Evans * executing prof_tctx_destroy(). 11704211e226628c41da4b3804ba411b5dd4b3a02abJason Evans */ 11804211e226628c41da4b3804ba411b5dd4b3a02abJason Evans uint64_t tctx_uid; 11904211e226628c41da4b3804ba411b5dd4b3a02abJason Evans 120602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans /* Linkage into gctx's tctxs. */ 121602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans rb_node(prof_tctx_t) tctx_link; 122602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 1236e73dc194ee9682d3eacaf725a989f04629718f7Jason Evans /* 1246e73dc194ee9682d3eacaf725a989f04629718f7Jason Evans * True during prof_alloc_prep()..prof_malloc_sample_object(), prevents 1256e73dc194ee9682d3eacaf725a989f04629718f7Jason Evans * sample vs destroy race. 1266e73dc194ee9682d3eacaf725a989f04629718f7Jason Evans */ 1276e73dc194ee9682d3eacaf725a989f04629718f7Jason Evans bool prepared; 1286e73dc194ee9682d3eacaf725a989f04629718f7Jason Evans 129602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans /* Current dump-related state, protected by gctx->lock. */ 130602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tctx_state_t state; 1316109fe07a14b7a619365977d9523db9f8b333792Jason Evans 1326109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* 133602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * Copy of cnts snapshotted during early dump phase, protected by 134602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * dump_mtx. 1356109fe07a14b7a619365977d9523db9f8b333792Jason Evans */ 136602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_cnt_t dump_cnts; 1376109fe07a14b7a619365977d9523db9f8b333792Jason Evans}; 138602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evanstypedef rb_tree(prof_tctx_t) prof_tctx_tree_t; 1396109fe07a14b7a619365977d9523db9f8b333792Jason Evans 140602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansstruct prof_gctx_s { 141602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans /* Protects nlimbo, cnt_summed, and tctxs. */ 1426da5418ded9170b087c35960e0010006430117c1Jason Evans malloc_mutex_t *lock; 1436109fe07a14b7a619365977d9523db9f8b333792Jason Evans 14452386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans /* 145602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * Number of threads that currently cause this gctx to be in a state of 14652386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans * limbo due to one of: 147602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * - Initializing this gctx. 148602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * - Initializing per thread counters associated with this gctx. 149602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * - Preparing to destroy this gctx. 150602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * - Dumping a heap profile that includes this gctx. 15152386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans * nlimbo must be 1 (single destroyer) in order to safely destroy the 152602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * gctx. 15352386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans */ 15452386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans unsigned nlimbo; 15552386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans 1566109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* 1573a81cbd2d4f2d8c052f11f4b0b73ee5c84a33d4fJason Evans * Tree of profile counters, one for each thread that has allocated in 1586109fe07a14b7a619365977d9523db9f8b333792Jason Evans * this context. 1596109fe07a14b7a619365977d9523db9f8b333792Jason Evans */ 160602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tctx_tree_t tctxs; 1614f37ef693e3d5903ce07dc0b61c0da320b35e3d9Jason Evans 1623a81cbd2d4f2d8c052f11f4b0b73ee5c84a33d4fJason Evans /* Linkage for tree of contexts to be dumped. */ 163602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans rb_node(prof_gctx_t) dump_link; 164602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 165602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans /* Temporary storage for summation during dump. */ 166602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_cnt_t cnt_summed; 167ab532e97991d190e9368781cf308c60c2319b933Jason Evans 168ab532e97991d190e9368781cf308c60c2319b933Jason Evans /* Associated backtrace. */ 169ab532e97991d190e9368781cf308c60c2319b933Jason Evans prof_bt_t bt; 170ab532e97991d190e9368781cf308c60c2319b933Jason Evans 171ab532e97991d190e9368781cf308c60c2319b933Jason Evans /* Backtrace vector, variable size, referred to by bt. */ 172ab532e97991d190e9368781cf308c60c2319b933Jason Evans void *vec[1]; 1736109fe07a14b7a619365977d9523db9f8b333792Jason Evans}; 174602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evanstypedef rb_tree(prof_gctx_t) prof_gctx_tree_t; 175602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 1764d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evansstruct prof_tdata_s { 177602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans malloc_mutex_t *lock; 178602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 179602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans /* Monotonically increasing unique thread identifier. */ 180602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans uint64_t thr_uid; 181602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 18220c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evans /* 18320c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evans * Monotonically increasing discriminator among tdata structures 18420c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evans * associated with the same thr_uid. 18520c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evans */ 18620c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evans uint64_t thr_discrim; 18720c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evans 188602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans /* Included in heap profile dumps if non-NULL. */ 189602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans char *thread_name; 190602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 19120c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evans bool attached; 19220c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evans bool expired; 193602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 194602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans rb_node(prof_tdata_t) tdata_link; 195602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 1964d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans /* 19704211e226628c41da4b3804ba411b5dd4b3a02abJason Evans * Counter used to initialize prof_tctx_t's tctx_uid. No locking is 19804211e226628c41da4b3804ba411b5dd4b3a02abJason Evans * necessary when incrementing this field, because only one thread ever 19904211e226628c41da4b3804ba411b5dd4b3a02abJason Evans * does so. 20004211e226628c41da4b3804ba411b5dd4b3a02abJason Evans */ 20104211e226628c41da4b3804ba411b5dd4b3a02abJason Evans uint64_t tctx_uid_next; 20204211e226628c41da4b3804ba411b5dd4b3a02abJason Evans 20304211e226628c41da4b3804ba411b5dd4b3a02abJason Evans /* 204602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * Hash of (prof_bt_t *)-->(prof_tctx_t *). Each thread tracks 205b41ccdb125b312d4522da1a80091a0137773c964Jason Evans * backtraces for which it has non-zero allocation/deallocation counters 206602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * associated with thread-specific prof_tctx_t objects. Other threads 207602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * may write to prof_tctx_t contents when freeing associated objects. 2084d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans */ 209602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans ckh_t bt2tctx; 210a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans 2114d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans /* Sampling state. */ 21284f7cdb0c588322dfd50a26497fc1cb54b792018Jason Evans uint64_t prng_state; 2136c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer uint64_t bytes_until_sample; 21452386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans 21552386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans /* State used to avoid dumping while operating on prof internals. */ 21652386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans bool enq; 21752386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans bool enq_idump; 21852386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans bool enq_gdump; 219b41ccdb125b312d4522da1a80091a0137773c964Jason Evans 220602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans /* 221602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * Set to true during an early dump phase for tdata's which are 222602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * currently being dumped. New threads' tdata's have this initialized 223602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * to false so that they aren't accidentally included in later dump 224602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * phases. 225602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans */ 226602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans bool dumping; 227602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 228602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans /* 229602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * True if profiling is active for this tdata's thread 230602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * (thread.prof.active mallctl). 231602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans */ 232602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans bool active; 233602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 234602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans /* Temporary storage for summation during dump. */ 235602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_cnt_t cnt_summed; 236602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 237b41ccdb125b312d4522da1a80091a0137773c964Jason Evans /* Backtrace vector, used for calls to prof_backtrace(). */ 238b41ccdb125b312d4522da1a80091a0137773c964Jason Evans void *vec[PROF_BT_MAX]; 239a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans}; 240602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evanstypedef rb_tree(prof_tdata_t) prof_tdata_tree_t; 241a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evans 2426109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif /* JEMALLOC_H_STRUCTS */ 2436109fe07a14b7a619365977d9523db9f8b333792Jason Evans/******************************************************************************/ 2446109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_H_EXTERNS 2456109fe07a14b7a619365977d9523db9f8b333792Jason Evans 2466109fe07a14b7a619365977d9523db9f8b333792Jason Evansextern bool opt_prof; 247f18c98200145de70779a1b3286e7829b0268231eJason Evansextern bool opt_prof_active; 248fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evansextern bool opt_prof_thread_active_init; 249a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evansextern size_t opt_lg_prof_sample; /* Mean bytes between samples. */ 250a02fc08ec9dd8479a6430155b6a433da09f6ff10Jason Evansextern ssize_t opt_lg_prof_interval; /* lg(prof_interval). */ 251e73397062ac3ab28a9d377591b63ed19fd154ccaJason Evansextern bool opt_prof_gdump; /* High-water memory dumping. */ 2520b25fe79aaf8840a5acda7e3160a053d42349872Jason Evansextern bool opt_prof_final; /* Final profile dumping. */ 253a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evansextern bool opt_prof_leak; /* Dump leak summary at exit. */ 254a881cd2c61c1ced56f87fcb9d7ef6e92b81e6c58Jason Evansextern bool opt_prof_accum; /* Report cumulative bytes. */ 255eefdd02e70ec1b9cf11920fcff585835dcbd766bJason Evansextern char opt_prof_prefix[ 256eefdd02e70ec1b9cf11920fcff585835dcbd766bJason Evans /* Minimize memory bloat for non-prof builds. */ 257eefdd02e70ec1b9cf11920fcff585835dcbd766bJason Evans#ifdef JEMALLOC_PROF 258eefdd02e70ec1b9cf11920fcff585835dcbd766bJason Evans PATH_MAX + 259eefdd02e70ec1b9cf11920fcff585835dcbd766bJason Evans#endif 260eefdd02e70ec1b9cf11920fcff585835dcbd766bJason Evans 1]; 2616109fe07a14b7a619365977d9523db9f8b333792Jason Evans 262fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans/* Accessed via prof_active_[gs]et{_unlocked,}(). */ 263fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evansextern bool prof_active; 264fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans 2655b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evans/* Accessed via prof_gdump_[gs]et{_unlocked,}(). */ 2665b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evansextern bool prof_gdump_val; 2675b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evans 268d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans/* 269d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans * Profile dump interval, measured in bytes allocated. Each arena triggers a 270d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans * profile dump when it reaches this threshold. The effect is that the 271d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans * interval between profile dumps averages prof_interval, though the actual 272d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans * interval between dumps will tend to be sporadic, and the interval will be a 273d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans * maximum of approximately (prof_interval * narenas). 274d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans */ 275d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evansextern uint64_t prof_interval; 276d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans 277602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans/* 278602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * Initialized as opt_lg_prof_sample, and potentially modified during profiling 279602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * resets. 280602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans */ 281602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansextern size_t lg_prof_sample; 282602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 2835460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansvoid prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated); 284c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize, 285602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tctx_t *tctx); 2865460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansvoid prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx); 2874d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evansvoid bt_init(prof_bt_t *bt, void **vec); 2886f001059aa33d77a3cb7799002044faf8dd08fc0Jason Evansvoid prof_backtrace(prof_bt_t *bt); 2895460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansprof_tctx_t *prof_lookup(tsd_t *tsd, prof_bt_t *bt); 290772163b4f3d8e9a12343e9215f6b070068507604Jason Evans#ifdef JEMALLOC_JET 29120c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evanssize_t prof_tdata_count(void); 292772163b4f3d8e9a12343e9215f6b070068507604Jason Evanssize_t prof_bt_count(void); 29320c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evansconst prof_cnt_t *prof_cnt_all(void); 294772163b4f3d8e9a12343e9215f6b070068507604Jason Evanstypedef int (prof_dump_open_t)(bool, const char *); 295772163b4f3d8e9a12343e9215f6b070068507604Jason Evansextern prof_dump_open_t *prof_dump_open; 296c1e00ef2a6442d1d047950247c757821560db329Jason Evanstypedef bool (prof_dump_header_t)(tsdn_t *, bool, const prof_cnt_t *); 29720c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evansextern prof_dump_header_t *prof_dump_header; 298772163b4f3d8e9a12343e9215f6b070068507604Jason Evans#endif 299c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid prof_idump(tsdn_t *tsdn); 300b2c0d6322d2307458ae2b28545f8a5c9903d7ef5Jason Evansbool prof_mdump(tsd_t *tsd, const char *filename); 301c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid prof_gdump(tsdn_t *tsdn); 302962a2979e353f876f3725417179f201e671d9dbbJason Evansprof_tdata_t *prof_tdata_init(tsd_t *tsd); 3035460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansprof_tdata_t *prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata); 304962a2979e353f876f3725417179f201e671d9dbbJason Evansvoid prof_reset(tsd_t *tsd, size_t lg_sample); 3055460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansvoid prof_tdata_cleanup(tsd_t *tsd); 306c1e00ef2a6442d1d047950247c757821560db329Jason Evansbool prof_active_get(tsdn_t *tsdn); 307c1e00ef2a6442d1d047950247c757821560db329Jason Evansbool prof_active_set(tsdn_t *tsdn, bool active); 308b2c0d6322d2307458ae2b28545f8a5c9903d7ef5Jason Evansconst char *prof_thread_name_get(tsd_t *tsd); 309fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evansint prof_thread_name_set(tsd_t *tsd, const char *thread_name); 310b2c0d6322d2307458ae2b28545f8a5c9903d7ef5Jason Evansbool prof_thread_active_get(tsd_t *tsd); 311b2c0d6322d2307458ae2b28545f8a5c9903d7ef5Jason Evansbool prof_thread_active_set(tsd_t *tsd, bool active); 312c1e00ef2a6442d1d047950247c757821560db329Jason Evansbool prof_thread_active_init_get(tsdn_t *tsdn); 313c1e00ef2a6442d1d047950247c757821560db329Jason Evansbool prof_thread_active_init_set(tsdn_t *tsdn, bool active_init); 314c1e00ef2a6442d1d047950247c757821560db329Jason Evansbool prof_gdump_get(tsdn_t *tsdn); 315c1e00ef2a6442d1d047950247c757821560db329Jason Evansbool prof_gdump_set(tsdn_t *tsdn, bool active); 3166109fe07a14b7a619365977d9523db9f8b333792Jason Evansvoid prof_boot0(void); 317e73397062ac3ab28a9d377591b63ed19fd154ccaJason Evansvoid prof_boot1(void); 318962a2979e353f876f3725417179f201e671d9dbbJason Evansbool prof_boot2(tsd_t *tsd); 319c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid prof_prefork0(tsdn_t *tsdn); 320c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid prof_prefork1(tsdn_t *tsdn); 321c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid prof_postfork_parent(tsdn_t *tsdn); 322c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid prof_postfork_child(tsdn_t *tsdn); 323602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansvoid prof_sample_threshold_update(prof_tdata_t *tdata); 3246109fe07a14b7a619365977d9523db9f8b333792Jason Evans 3256109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif /* JEMALLOC_H_EXTERNS */ 3266109fe07a14b7a619365977d9523db9f8b333792Jason Evans/******************************************************************************/ 3276109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_H_INLINES 3286109fe07a14b7a619365977d9523db9f8b333792Jason Evans 3294d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans#ifndef JEMALLOC_ENABLE_INLINE 330fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evansbool prof_active_get_unlocked(void); 3315b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evansbool prof_gdump_get_unlocked(void); 3325460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansprof_tdata_t *prof_tdata_get(tsd_t *tsd, bool create); 333c1e00ef2a6442d1d047950247c757821560db329Jason Evansprof_tctx_t *prof_tctx_get(tsdn_t *tsdn, const void *ptr); 334c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize, 335b2c0d6322d2307458ae2b28545f8a5c9903d7ef5Jason Evans prof_tctx_t *tctx); 336c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid prof_tctx_reset(tsdn_t *tsdn, const void *ptr, size_t usize, 337b2c0d6322d2307458ae2b28545f8a5c9903d7ef5Jason Evans const void *old_ptr, prof_tctx_t *tctx); 3385460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansbool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool commit, 339602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tdata_t **tdata_out); 340cec0d63d8bc46205d38456024176a0ece590253eJason Evansprof_tctx_t *prof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active, 341cec0d63d8bc46205d38456024176a0ece590253eJason Evans bool update); 342c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid prof_malloc(tsdn_t *tsdn, const void *ptr, size_t usize, 343602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tctx_t *tctx); 3445460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansvoid prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, 345708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans prof_tctx_t *tctx, bool prof_active, bool updated, const void *old_ptr, 346cec0d63d8bc46205d38456024176a0ece590253eJason Evans size_t old_usize, prof_tctx_t *old_tctx); 3475460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansvoid prof_free(tsd_t *tsd, const void *ptr, size_t usize); 3484d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans#endif 3494d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 3504d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_)) 35134e85b4182d5ae029b558aae3da25fff7c3efe12Jason EvansJEMALLOC_ALWAYS_INLINE bool 352fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evansprof_active_get_unlocked(void) 353fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans{ 354fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans 355fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans /* 356fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans * Even if opt_prof is true, sampling can be temporarily disabled by 357fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans * setting prof_active to false. No locking is used when reading 358fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans * prof_active in the fast path, so there are no guarantees regarding 359fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans * how long it will take for all threads to notice state changes. 360fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans */ 361fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans return (prof_active); 362fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans} 363fc12c0b8bc1160530d1e3e641b76d2a4f793136fJason Evans 3645b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason EvansJEMALLOC_ALWAYS_INLINE bool 3655b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evansprof_gdump_get_unlocked(void) 3665b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evans{ 3675b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evans 3685b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evans /* 3695b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evans * No locking is used when reading prof_gdump_val in the fast path, so 3705b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evans * there are no guarantees regarding how long it will take for all 3715b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evans * threads to notice state changes. 3725b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evans */ 3735b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evans return (prof_gdump_val); 3745b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evans} 3755b8ed5b7c91939f64f14fc48be84ed20e3f023f4Jason Evans 37634e85b4182d5ae029b558aae3da25fff7c3efe12Jason EvansJEMALLOC_ALWAYS_INLINE prof_tdata_t * 3775460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansprof_tdata_get(tsd_t *tsd, bool create) 37852386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans{ 379602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tdata_t *tdata; 38052386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans 38152386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans cassert(config_prof); 38252386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans 3835460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans tdata = tsd_prof_tdata_get(tsd); 384602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans if (create) { 3855460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans if (unlikely(tdata == NULL)) { 386029d44cf8b22aa7b749747bfd585887fb59e0030Jason Evans if (tsd_nominal(tsd)) { 387962a2979e353f876f3725417179f201e671d9dbbJason Evans tdata = prof_tdata_init(tsd); 388029d44cf8b22aa7b749747bfd585887fb59e0030Jason Evans tsd_prof_tdata_set(tsd, tdata); 389029d44cf8b22aa7b749747bfd585887fb59e0030Jason Evans } 39020c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evans } else if (unlikely(tdata->expired)) { 3915460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans tdata = prof_tdata_reinit(tsd, tdata); 3925460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans tsd_prof_tdata_set(tsd, tdata); 3935460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans } 39420c31deaae38ed9aa4fe169ed65e0c45cd542955Jason Evans assert(tdata == NULL || tdata->attached); 395602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans } 39652386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans 397602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans return (tdata); 39852386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans} 39952386b2dc689db3bf71307424c4e1a2b7044c363Jason Evans 40034e85b4182d5ae029b558aae3da25fff7c3efe12Jason EvansJEMALLOC_ALWAYS_INLINE prof_tctx_t * 401c1e00ef2a6442d1d047950247c757821560db329Jason Evansprof_tctx_get(tsdn_t *tsdn, const void *ptr) 4024d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans{ 4034d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 4047372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 4054d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans assert(ptr != NULL); 4064d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 407c1e00ef2a6442d1d047950247c757821560db329Jason Evans return (arena_prof_tctx_get(tsdn, ptr)); 4084d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans} 4094d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 41034e85b4182d5ae029b558aae3da25fff7c3efe12Jason EvansJEMALLOC_ALWAYS_INLINE void 411c1e00ef2a6442d1d047950247c757821560db329Jason Evansprof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize, prof_tctx_t *tctx) 4124d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans{ 4134d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 4147372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 4154d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans assert(ptr != NULL); 4164d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 417c1e00ef2a6442d1d047950247c757821560db329Jason Evans arena_prof_tctx_set(tsdn, ptr, usize, tctx); 4184d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans} 4194d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 420708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason EvansJEMALLOC_ALWAYS_INLINE void 421c1e00ef2a6442d1d047950247c757821560db329Jason Evansprof_tctx_reset(tsdn_t *tsdn, const void *ptr, size_t usize, const void *old_ptr, 422708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans prof_tctx_t *old_tctx) 423708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans{ 424708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans 425708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans cassert(config_prof); 426708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans assert(ptr != NULL); 427708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans 428c1e00ef2a6442d1d047950247c757821560db329Jason Evans arena_prof_tctx_reset(tsdn, ptr, usize, old_ptr, old_tctx); 429708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans} 430708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans 43134e85b4182d5ae029b558aae3da25fff7c3efe12Jason EvansJEMALLOC_ALWAYS_INLINE bool 4325460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansprof_sample_accum_update(tsd_t *tsd, size_t usize, bool update, 4335460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans prof_tdata_t **tdata_out) 4344d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans{ 435602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tdata_t *tdata; 4364d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 4377372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 4384d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 4395460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans tdata = prof_tdata_get(tsd, true); 440f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang if (unlikely((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)) 441602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans tdata = NULL; 4426c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer 443602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans if (tdata_out != NULL) 444602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans *tdata_out = tdata; 4456c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer 446f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang if (unlikely(tdata == NULL)) 447f27899402914065a6c1484ea8d81a2c8b70aa659Jason Evans return (true); 4484d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 449f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang if (likely(tdata->bytes_until_sample >= usize)) { 4506e73dc194ee9682d3eacaf725a989f04629718f7Jason Evans if (update) 451602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans tdata->bytes_until_sample -= usize; 4526c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer return (true); 4536c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer } else { 454ff7450727f64180367f430b1b747f9e682e26df4Jason Evans /* Compute new sample threshold. */ 4556e73dc194ee9682d3eacaf725a989f04629718f7Jason Evans if (update) 456602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_sample_threshold_update(tdata); 457551ebc43647521bdd0bc78558b106762b3388928Jason Evans return (!tdata->active); 4584d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans } 4594d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans} 4604d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 46134e85b4182d5ae029b558aae3da25fff7c3efe12Jason EvansJEMALLOC_ALWAYS_INLINE prof_tctx_t * 462cec0d63d8bc46205d38456024176a0ece590253eJason Evansprof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active, bool update) 463602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans{ 464602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tctx_t *ret; 465602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tdata_t *tdata; 466602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_bt_t bt; 467602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 468602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans assert(usize == s2u(usize)); 469602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 470cec0d63d8bc46205d38456024176a0ece590253eJason Evans if (!prof_active || likely(prof_sample_accum_update(tsd, usize, update, 471cec0d63d8bc46205d38456024176a0ece590253eJason Evans &tdata))) 472602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans ret = (prof_tctx_t *)(uintptr_t)1U; 473602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans else { 474602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans bt_init(&bt, tdata->vec); 475602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_backtrace(&bt); 4765460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans ret = prof_lookup(tsd, &bt); 4776c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer } 478602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans 479602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans return (ret); 4806c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer} 4816c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer 48234e85b4182d5ae029b558aae3da25fff7c3efe12Jason EvansJEMALLOC_ALWAYS_INLINE void 483c1e00ef2a6442d1d047950247c757821560db329Jason Evansprof_malloc(tsdn_t *tsdn, const void *ptr, size_t usize, prof_tctx_t *tctx) 4844d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans{ 4854d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 4867372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 4874d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans assert(ptr != NULL); 488c1e00ef2a6442d1d047950247c757821560db329Jason Evans assert(usize == isalloc(tsdn, ptr, true)); 4894d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 4909c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) 491c1e00ef2a6442d1d047950247c757821560db329Jason Evans prof_malloc_sample_object(tsdn, ptr, usize, tctx); 4926c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer else 493c1e00ef2a6442d1d047950247c757821560db329Jason Evans prof_tctx_set(tsdn, ptr, usize, (prof_tctx_t *)(uintptr_t)1U); 4944d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans} 4954d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 49634e85b4182d5ae029b558aae3da25fff7c3efe12Jason EvansJEMALLOC_ALWAYS_INLINE void 4975460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansprof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx, 498708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans bool prof_active, bool updated, const void *old_ptr, size_t old_usize, 499708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans prof_tctx_t *old_tctx) 5004d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans{ 501ea8d97b8978a0c0423f0ed64332463a25b787c3dJason Evans bool sampled, old_sampled; 5024d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 5037372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 504602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans assert(ptr != NULL || (uintptr_t)tctx <= (uintptr_t)1U); 5054d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 506cec0d63d8bc46205d38456024176a0ece590253eJason Evans if (prof_active && !updated && ptr != NULL) { 507c1e00ef2a6442d1d047950247c757821560db329Jason Evans assert(usize == isalloc(tsd_tsdn(tsd), ptr, true)); 5085460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans if (prof_sample_accum_update(tsd, usize, true, NULL)) { 5096c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer /* 510cec0d63d8bc46205d38456024176a0ece590253eJason Evans * Don't sample. The usize passed to prof_alloc_prep() 511602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * was larger than what actually got allocated, so a 512602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * backtrace was captured for this allocation, even 513602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * though its actual usize was insufficient to cross the 514602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans * sample threshold. 5156c39f9e059d0825f4c29d8cec9f318b798912c3cBen Maurer */ 51620cd2de5ef622c3af8b3e4aba897aff7ddd451a7Jason Evans prof_alloc_rollback(tsd, tctx, true); 517602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans tctx = (prof_tctx_t *)(uintptr_t)1U; 5184d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans } 5194d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans } 5204d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 521ea8d97b8978a0c0423f0ed64332463a25b787c3dJason Evans sampled = ((uintptr_t)tctx > (uintptr_t)1U); 522ea8d97b8978a0c0423f0ed64332463a25b787c3dJason Evans old_sampled = ((uintptr_t)old_tctx > (uintptr_t)1U); 523ea8d97b8978a0c0423f0ed64332463a25b787c3dJason Evans 524ea8d97b8978a0c0423f0ed64332463a25b787c3dJason Evans if (unlikely(sampled)) 525c1e00ef2a6442d1d047950247c757821560db329Jason Evans prof_malloc_sample_object(tsd_tsdn(tsd), ptr, usize, tctx); 526602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans else 527c1e00ef2a6442d1d047950247c757821560db329Jason Evans prof_tctx_reset(tsd_tsdn(tsd), ptr, usize, old_ptr, old_tctx); 528ea8d97b8978a0c0423f0ed64332463a25b787c3dJason Evans 529ea8d97b8978a0c0423f0ed64332463a25b787c3dJason Evans if (unlikely(old_sampled)) 530ea8d97b8978a0c0423f0ed64332463a25b787c3dJason Evans prof_free_sampled_object(tsd, old_usize, old_tctx); 5314d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans} 5324d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 53334e85b4182d5ae029b558aae3da25fff7c3efe12Jason EvansJEMALLOC_ALWAYS_INLINE void 5345460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansprof_free(tsd_t *tsd, const void *ptr, size_t usize) 5354d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans{ 536c1e00ef2a6442d1d047950247c757821560db329Jason Evans prof_tctx_t *tctx = prof_tctx_get(tsd_tsdn(tsd), ptr); 5374d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 5387372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 539c1e00ef2a6442d1d047950247c757821560db329Jason Evans assert(usize == isalloc(tsd_tsdn(tsd), ptr, true)); 5407372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans 5419c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) 5425460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans prof_free_sampled_object(tsd, usize, tctx); 5434d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans} 5444d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans#endif 5454d6a134e13e1e9705faff6f5dc6ca90d416aa0a4Jason Evans 5466109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif /* JEMALLOC_H_INLINES */ 5476109fe07a14b7a619365977d9523db9f8b333792Jason Evans/******************************************************************************/ 548