1ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--------------------------------------------------------------------*/ 3ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--- LibHB: a library for implementing and checking ---*/ 4ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--- the happens-before relationship in concurrent programs. ---*/ 5ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--- libhb_main.c ---*/ 6ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--------------------------------------------------------------------*/ 7ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 8ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* 9ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown This file is part of LibHB, a library for implementing and checking 10ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown the happens-before relationship in concurrent programs. 11ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 12436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov Copyright (C) 2008-2013 OpenWorks Ltd 13ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown info@open-works.co.uk 14ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 15ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown This program is free software; you can redistribute it and/or 16ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown modify it under the terms of the GNU General Public License as 17ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown published by the Free Software Foundation; either version 2 of the 18ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown License, or (at your option) any later version. 19ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 20ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown This program is distributed in the hope that it will be useful, but 21ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown WITHOUT ANY WARRANTY; without even the implied warranty of 22ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 23ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown General Public License for more details. 24ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 25ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown You should have received a copy of the GNU General Public License 26ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown along with this program; if not, write to the Free Software 27ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 28ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 02111-1307, USA. 29ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 30ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown The GNU General Public License is contained in the file COPYING. 31ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 32ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 33ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_basics.h" 34663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng#include "pub_tool_poolalloc.h" 35ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_libcassert.h" 36ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_libcbase.h" 37ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_libcprint.h" 38ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_mallocfree.h" 39ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_wordfm.h" 40ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_sparsewa.h" 41ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_xarray.h" 42ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_oset.h" 43ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_threadstate.h" 44ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_aspacemgr.h" 45ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_execontext.h" 46ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_errormgr.h" 47ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_options.h" // VG_(clo_stats) 48ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "hg_basics.h" 49ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "hg_wordset.h" 50ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "hg_lock_n_thread.h" 51ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "hg_errors.h" 52ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 53ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "libhb.h" 54ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 55ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 56ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 57ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 58ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 59ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Debugging #defines // 60ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 61ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 62ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 63ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 64ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Check the sanity of shadow values in the core memory state 65ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown machine. Change #if 0 to #if 1 to enable this. */ 66ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#if 0 67ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# define CHECK_MSM 1 68ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#else 69ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# define CHECK_MSM 0 70ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#endif 71ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 72ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 73ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Check sanity (reference counts, etc) in the conflicting access 74ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown machinery. Change #if 0 to #if 1 to enable this. */ 75ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#if 0 76ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# define CHECK_CEM 1 77ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#else 78ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# define CHECK_CEM 0 79ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#endif 80ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 81ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 82ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Check sanity in the compressed shadow memory machinery, 83ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown particularly in its caching innards. Unfortunately there's no 84ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown almost-zero-cost way to make them selectable at run time. Hence 85ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown set the #if 0 to #if 1 and rebuild if you want them. */ 86ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#if 0 87ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# define CHECK_ZSM 1 /* do sanity-check CacheLine stuff */ 88ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# define inline __attribute__((noinline)) 89ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* probably want to ditch -fomit-frame-pointer too */ 90ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#else 91ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# define CHECK_ZSM 0 /* don't sanity-check CacheLine stuff */ 92ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#endif 93ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 94ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 95ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 96ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 97ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 98b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// data decls: VtsID // 99ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 100ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 101ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 102ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 103b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* VtsIDs: Unique small-integer IDs for VTSs. VtsIDs can't exceed 30 104b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov bits, since they have to be packed into the lowest 30 bits of an 105b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov SVal. */ 106b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovtypedef UInt VtsID; 107b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define VtsID_INVALID 0xFFFFFFFF 108ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 109ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 110ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 111ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 112ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 113ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 114b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// data decls: SVal // 115ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 116ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 117ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 118ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 119ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef ULong SVal; 120ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 121ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* This value has special significance to the implementation, and callers 122ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown may not store it in the shadow memory. */ 123ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define SVal_INVALID (3ULL << 62) 124ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 125ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* This is the default value for shadow memory. Initially the shadow 126ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown memory contains no accessible areas and so all reads produce this 127ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown value. TODO: make this caller-defineable. */ 128ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define SVal_NOACCESS (2ULL << 62) 129ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 130b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 131b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 132b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 133b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 134b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// // 135b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// data decls: ScalarTS // 136b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// // 137b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 138b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 139b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 140b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Scalar Timestamp. We have to store a lot of these, so there is 141b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov some effort to make them as small as possible. Logically they are 142b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a pair, (Thr*, ULong), but that takes 16 bytes on a 64-bit target. 143b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov We pack it into 64 bits by representing the Thr* using a ThrID, a 144b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov small integer (18 bits), and a 46 bit integer for the timestamp 145b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov number. The 46/18 split is arbitary, but has the effect that 146b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Helgrind can only handle programs that create 2^18 or fewer threads 147b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov over their entire lifetime, and have no more than 2^46 timestamp 148b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ticks (synchronisation operations on the same thread). 149b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 150b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov This doesn't seem like much of a limitation. 2^46 ticks is 151b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 7.06e+13, and if each tick (optimistically) takes the machine 1000 152b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cycles to process, then the minimum time to process that many ticks 153b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov at a clock rate of 5 GHz is 162.9 days. And that's doing nothing 154b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov but VTS ticks, which isn't realistic. 155b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 156b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov NB1: SCALARTS_N_THRBITS must be 29 or lower. The obvious limit is 157b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 32 since a ThrID is a UInt. 29 comes from the fact that 158b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 'Thr_n_RCEC', which records information about old accesses, packs 159b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov not only a ThrID but also 2+1 other bits (access size and 160b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov writeness) in a UInt, hence limiting size to 32-(2+1) == 29. 161b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 162b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov NB2: thrid values are issued upwards from 1024, and values less 163b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov than that aren't valid. This isn't per se necessary (any order 164b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov will do, so long as they are unique), but it does help ensure they 165b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov are less likely to get confused with the various other kinds of 166b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov small-integer thread ids drifting around (eg, TId). See also NB5. 167b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 168b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov NB3: this probably also relies on the fact that Thr's are never 169b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov deallocated -- they exist forever. Hence the 1-1 mapping from 170b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Thr's to thrid values (set up in Thr__new) persists forever. 171b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 172b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov NB4: temp_max_sized_VTS is allocated at startup and never freed. 173b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov It is a maximum sized VTS, so has (1 << SCALARTS_N_TYMBITS) 174b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTSs. So we can't make SCALARTS_N_THRBITS too large without 175b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov making the memory use for this go sky-high. With 176b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov SCALARTS_N_THRBITS at 18, it occupies 2MB of memory, which seems 177b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov like an OK tradeoff. If more than 256k threads need to be 178b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov supported, we could change SCALARTS_N_THRBITS to 20, which would 179b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov facilitate supporting 1 million threads at the cost of 8MB storage 180b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for temp_max_sized_VTS. 181b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 182b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov NB5: the conflicting-map mechanism (Thr_n_RCEC, specifically) uses 183b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID == 0 to denote an empty Thr_n_RCEC record. So ThrID == 0 184b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov must never be a valid ThrID. Given NB2 that's OK. 185b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov*/ 186b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define SCALARTS_N_THRBITS 18 /* valid range: 11 to 29 inclusive */ 187b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 188b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define SCALARTS_N_TYMBITS (64 - SCALARTS_N_THRBITS) 189b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovtypedef 190b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov struct { 191b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID thrid : SCALARTS_N_THRBITS; 192b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ULong tym : SCALARTS_N_TYMBITS; 193b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 194b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS; 195b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 196b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define ThrID_MAX_VALID ((1 << SCALARTS_N_THRBITS) - 1) 197b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 198b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 199b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 200b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 201b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 202b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// // 203b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// data decls: Filter // 204b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// // 205b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 206b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 207b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 208b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// baseline: 5, 9 209b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define FI_LINE_SZB_LOG2 5 210b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define FI_NUM_LINES_LOG2 10 211b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 212b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define FI_LINE_SZB (1 << FI_LINE_SZB_LOG2) 213b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define FI_NUM_LINES (1 << FI_NUM_LINES_LOG2) 214b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 215b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define FI_TAG_MASK (~(Addr)(FI_LINE_SZB - 1)) 216b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define FI_GET_TAG(_a) ((_a) & FI_TAG_MASK) 217b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 218b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define FI_GET_LINENO(_a) ( ((_a) >> FI_LINE_SZB_LOG2) \ 219b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov & (Addr)(FI_NUM_LINES-1) ) 220b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 221b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 222b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* In the lines, each 8 bytes are treated individually, and are mapped 223b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov to a UShort. Regardless of endianness of the underlying machine, 224b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov bits 1 and 0 pertain to the lowest address and bits 15 and 14 to 225b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov the highest address. 226b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 227b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Of each bit pair, the higher numbered bit is set if a R has been 228b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov seen, so the actual layout is: 229b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 230b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 15 14 ... 01 00 231b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 232b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov R W for addr+7 ... R W for addr+0 233b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 234b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov So a mask for the R-bits is 0xAAAA and for the W bits is 0x5555. 235b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov*/ 236b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 237b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* tags are separated from lines. tags are Addrs and are 238b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov the base address of the line. */ 239b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovtypedef 240b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov struct { 241b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UShort u16s[FI_LINE_SZB / 8]; /* each UShort covers 8 bytes */ 242b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 243b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov FiLine; 244b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 245b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovtypedef 246b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov struct { 247b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Addr tags[FI_NUM_LINES]; 248b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov FiLine lines[FI_NUM_LINES]; 249b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 250b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Filter; 251b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 252b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 253b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 254b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 255b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 256b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// // 257b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// data decls: Thr, ULong_n_EC // 258b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// // 259b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 260b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 261b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 262b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// Records stacks for H1 history mechanism (DRD-style) 263b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovtypedef 264b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov struct { ULong ull; ExeContext* ec; } 265b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ULong_n_EC; 266b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 267b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 268b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* How many of the above records to collect for each thread? Older 269b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ones are dumped when we run out of space. 62.5k requires 1MB per 270b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thread, since each ULong_n_EC record is 16 bytes long. When more 271b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov than N_KWs_N_STACKs_PER_THREAD are present, the older half are 272b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov deleted to make space. Hence in the worst case we will be able to 273b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov produce a stack at least for the last N_KWs_N_STACKs_PER_THREAD / 2 274b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Kw transitions (segments in this thread). For the current setting 275b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov that gives a guaranteed stack for at least the last 31.25k 276b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov segments. */ 277b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define N_KWs_N_STACKs_PER_THREAD 62500 278b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 279b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 280b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstruct _Thr { 281b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Current VTSs for this thread. They change as we go along. viR 282b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov is the VTS to be used for reads, viW for writes. Usually they 283b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov are the same, but can differ when we deal with reader-writer 284b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov locks. It is always the case that 285b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID__cmpLEQ(viW,viR) == True 286b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov that is, viW must be the same, or lagging behind, viR. */ 287b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID viR; 288b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID viW; 289b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 290b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Is initially False, and is set to True after the thread really 291b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov has done a low-level exit. When True, we expect to never see 292b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov any more memory references done by this thread. */ 293b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Bool llexit_done; 294b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 295b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Is initially False, and is set to True after the thread has been 296b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov joined with (reaped by some other thread). After this point, we 297b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov do not expect to see any uses of .viR or .viW, so it is safe to 298b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov set them to VtsID_INVALID. */ 299b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Bool joinedwith_done; 300b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 301b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* A small integer giving a unique identity to this Thr. See 302b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov comments on the definition of ScalarTS for details. */ 303b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID thrid : SCALARTS_N_THRBITS; 304b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 305b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* A filter that removes references for which we believe that 306b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov msmcread/msmcwrite will not change the state, nor report a 307b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov race. */ 308b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Filter* filter; 309b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 310b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* A pointer back to the top level Thread structure. There is a 311b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1-1 mapping between Thread and Thr structures -- each Thr points 312b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov at its corresponding Thread, and vice versa. Really, Thr and 313b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Thread should be merged into a single structure. */ 314b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Thread* hgthread; 315b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 316b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* The ULongs (scalar Kws) in this accumulate in strictly 317b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov increasing order, without duplicates. This is important because 318b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov we need to be able to find a given scalar Kw in this array 319b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov later, by binary search. */ 320b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov XArray* /* ULong_n_EC */ local_Kws_n_stacks; 321b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov}; 322b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 323b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 324b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 325b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 326b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 327b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// // 328b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// data decls: SO // 329b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// // 330b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 331b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 332b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 333b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// (UInt) `echo "Synchronisation object" | md5sum` 334b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define SO_MAGIC 0x56b3c5b0U 335b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 336b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstruct _SO { 337b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov struct _SO* admin_prev; 338b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov struct _SO* admin_next; 339b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID viR; /* r-clock of sender */ 340b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID viW; /* w-clock of sender */ 341b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt magic; 342b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov}; 343b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 344b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 345b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 346b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 347b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 348b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// // 349b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// Forward declarations // 350b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// // 351b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 352b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 353b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 354b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* fwds for 355b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Globals needed by other parts of the library. These are set 356b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov once at startup and then never changed. */ 357b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void (*main_get_stacktrace)( Thr*, Addr*, UWord ) = NULL; 358b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic ExeContext* (*main_get_EC)( Thr* ) = NULL; 359b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 360b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* misc fn and data fwdses */ 361b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void VtsID__rcinc ( VtsID ii ); 362b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void VtsID__rcdec ( VtsID ii ); 363b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 364b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic inline Bool SVal__isC ( SVal s ); 365b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic inline VtsID SVal__unC_Rmin ( SVal s ); 366b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic inline VtsID SVal__unC_Wmin ( SVal s ); 367b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic inline SVal SVal__mkC ( VtsID rmini, VtsID wmini ); 368b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 369b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* A double linked list of all the SO's. */ 370b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy StepanovSO* admin_SO; 371b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 372b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 373b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 374b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 375b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 376b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// // 377b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// SECTION BEGIN compressed shadow memory // 378b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// // 379b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 380b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 381b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 382b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#ifndef __HB_ZSM_H 383b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define __HB_ZSM_H 384b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 385ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Initialise the library. Once initialised, it will (or may) call 386ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcinc and rcdec in response to all the calls below, in order to 387ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown allow the user to do reference counting on the SVals stored herein. 388ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown It is important to understand, however, that due to internal 389ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown caching, the reference counts are in general inaccurate, and can be 390ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown both above or below the true reference count for an item. In 391ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown particular, the library may indicate that the reference count for 392ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown an item is zero, when in fact it is not. 393ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 394ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown To make the reference counting exact and therefore non-pointless, 395ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown call zsm_flush_cache. Immediately after it returns, the reference 396ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown counts for all items, as deduced by the caller by observing calls 397ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown to rcinc and rcdec, will be correct, and so any items with a zero 398ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown reference count may be freed (or at least considered to be 399ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown unreferenced by this library). 400ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 401ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_init ( void(*rcinc)(SVal), void(*rcdec)(SVal) ); 402ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 403ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_sset_range ( Addr, SizeT, SVal ); 404ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_scopy_range ( Addr, Addr, SizeT ); 405ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_flush_cache ( void ); 406ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 407ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#endif /* ! __HB_ZSM_H */ 408ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 409ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 410ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Round a up to the next multiple of N. N must be a power of 2 */ 411ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define ROUNDUP(a, N) ((a + N - 1) & ~(N-1)) 412ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Round a down to the next multiple of N. N must be a power of 2 */ 413ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define ROUNDDN(a, N) ((a) & ~(N-1)) 414ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 415ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 416ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 417ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* ------ User-supplied RC functions ------ */ 418ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void(*rcinc)(SVal) = NULL; 419ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void(*rcdec)(SVal) = NULL; 420ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 421ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 422ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* ------ CacheLine ------ */ 423ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 424ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_LINE_BITS 6 /* must be >= 3 */ 425ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_LINE_ARANGE (1 << N_LINE_BITS) 426ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_LINE_TREES (N_LINE_ARANGE >> 3) 427ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 428ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef 429ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct { 430ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descrs[N_LINE_TREES]; 431ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal svals[N_LINE_ARANGE]; // == N_LINE_TREES * 8 432ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 433ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine; 434ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 435ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_16_0 (1<<0) 436ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_32_0 (1<<1) 437ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_16_1 (1<<2) 438ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_64 (1<<3) 439ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_16_2 (1<<4) 440ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_32_1 (1<<5) 441ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_16_3 (1<<6) 442ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_8_0 (1<<7) 443ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_8_1 (1<<8) 444ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_8_2 (1<<9) 445ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_8_3 (1<<10) 446ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_8_4 (1<<11) 447ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_8_5 (1<<12) 448ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_8_6 (1<<13) 449ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_8_7 (1<<14) 450ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_DTY (1<<15) 451ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 452ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef 453ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct { 454ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal dict[4]; /* can represent up to 4 diff values in the line */ 455ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UChar ix2s[N_LINE_ARANGE/4]; /* array of N_LINE_ARANGE 2-bit 456ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown dict indexes */ 457ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* if dict[0] == SVal_INVALID then dict[1] is the index of the 458ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineF to use, and dict[2..] are also SVal_INVALID. */ 459ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 460ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineZ; /* compressed rep for a cache line */ 461ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 462ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef 463ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct { 464ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool inUse; 465ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal w64s[N_LINE_ARANGE]; 466ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 467ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineF; /* full rep for a cache line */ 468ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 469ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Shadow memory. 470ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Primary map is a WordFM Addr SecMap*. 471ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SecMaps cover some page-size-ish section of address space and hold 472ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a compressed representation. 473ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine-sized chunks of SecMaps are copied into a Cache, being 474ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown decompressed when moved into the cache and recompressed on the 475ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown way out. Because of this, the cache must operate as a writeback 476ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cache, not a writethrough one. 477ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 478ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Each SecMap must hold a power-of-2 number of CacheLines. Hence 479ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown N_SECMAP_BITS must >= N_LINE_BITS. 480ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 481ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_SECMAP_BITS 13 482ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_SECMAP_ARANGE (1 << N_SECMAP_BITS) 483ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 484ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// # CacheLines held by a SecMap 485ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_SECMAP_ZLINES (N_SECMAP_ARANGE / N_LINE_ARANGE) 486ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 487ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* The data in the SecMap is held in the array of LineZs. Each LineZ 488ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown either carries the required data directly, in a compressed 489ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown representation, or it holds (in .dict[0]) an index to the LineF in 490ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown .linesF that holds the full representation. 491ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 492ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Currently-unused LineF's have their .inUse bit set to zero. 493ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Since each in-use LineF is referred to be exactly one LineZ, 494ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown the number of .linesZ[] that refer to .linesF should equal 495ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown the number of .linesF[] that have .inUse == True. 496ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 497ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown RC obligations: the RCs presented to the user include exactly 498ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown the values in: 499ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown * direct Z reps, that is, ones for which .dict[0] != SVal_INVALID 500ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown * F reps that are in use (.inUse == True) 501ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 502ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Hence the following actions at the following transitions are required: 503ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 504ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown F rep: .inUse==True -> .inUse==False -- rcdec_LineF 505ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown F rep: .inUse==False -> .inUse==True -- rcinc_LineF 506ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Z rep: .dict[0] from other to SVal_INVALID -- rcdec_LineZ 507ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Z rep: .dict[0] from SVal_INVALID to other -- rcinc_LineZ 508ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 509ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef 510ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct { 511ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt magic; 512ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineZ linesZ[N_SECMAP_ZLINES]; 513ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineF* linesF; 514ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt linesF_size; 515ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 516ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SecMap; 517ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 518ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define SecMap_MAGIC 0x571e58cbU 519ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 520ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool is_sane_SecMap ( SecMap* sm ) { 521ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return sm != NULL && sm->magic == SecMap_MAGIC; 522ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 523ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 524ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* ------ Cache ------ */ 525ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 526ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_WAY_BITS 16 527ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_WAY_NENT (1 << N_WAY_BITS) 528ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 529ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Each tag is the address of the associated CacheLine, rounded down 530ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown to a CacheLine address boundary. A CacheLine size must be a power 531ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown of 2 and must be 8 or more. Hence an easy way to initialise the 532ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cache so it is empty is to set all the tag values to any value % 8 533ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown != 0, eg 1. This means all queries in the cache initially miss. 534ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown It does however require us to detect and not writeback, any line 535ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown with a bogus tag. */ 536ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef 537ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct { 538ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine lyns0[N_WAY_NENT]; 539ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr tags0[N_WAY_NENT]; 540ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 541ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Cache; 542ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 543ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool is_valid_scache_tag ( Addr tag ) { 544ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* a valid tag should be naturally aligned to the start of 545ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a CacheLine. */ 546ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0 == (tag & (N_LINE_ARANGE - 1)); 547ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 548ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 549ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 550ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* --------- Primary data structures --------- */ 551ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 552ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Shadow memory primary map */ 553ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic WordFM* map_shmem = NULL; /* WordFM Addr SecMap* */ 554ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Cache cache_shmem; 555ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 556ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 557ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__secmaps_search = 0; // # SM finds 558ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__secmaps_search_slow = 0; // # SM lookupFMs 559ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__secmaps_allocd = 0; // # SecMaps issued 560ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__secmap_ga_space_covered = 0; // # ga bytes covered 561ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__secmap_linesZ_allocd = 0; // # LineZ's issued 562ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__secmap_linesZ_bytes = 0; // .. using this much storage 563ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__secmap_linesF_allocd = 0; // # LineF's issued 564ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__secmap_linesF_bytes = 0; // .. using this much storage 565ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__secmap_iterator_steppings = 0; // # calls to stepSMIter 566ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cache_Z_fetches = 0; // # Z lines fetched 567ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cache_Z_wbacks = 0; // # Z lines written back 568ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cache_F_fetches = 0; // # F lines fetched 569ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cache_F_wbacks = 0; // # F lines written back 570ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cache_invals = 0; // # cache invals 571ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cache_flushes = 0; // # cache flushes 572ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cache_totrefs = 0; // # total accesses 573ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cache_totmisses = 0; // # misses 574ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic ULong stats__cache_make_New_arange = 0; // total arange made New 575ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic ULong stats__cache_make_New_inZrep = 0; // arange New'd on Z reps 576ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_normalises = 0; // # calls to cacheline_normalise 577ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_cread64s = 0; // # calls to s_m_read64 578ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_cread32s = 0; // # calls to s_m_read32 579ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_cread16s = 0; // # calls to s_m_read16 580ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_cread08s = 0; // # calls to s_m_read8 581ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_cwrite64s = 0; // # calls to s_m_write64 582ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_cwrite32s = 0; // # calls to s_m_write32 583ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_cwrite16s = 0; // # calls to s_m_write16 584ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_cwrite08s = 0; // # calls to s_m_write8 585ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_sread08s = 0; // # calls to s_m_set8 586ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_swrite08s = 0; // # calls to s_m_get8 587ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_swrite16s = 0; // # calls to s_m_get8 588ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_swrite32s = 0; // # calls to s_m_get8 589ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_swrite64s = 0; // # calls to s_m_get8 590ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_scopy08s = 0; // # calls to s_m_copy8 591ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_64to32splits = 0; // # 64-bit accesses split 592ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_32to16splits = 0; // # 32-bit accesses split 593ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_16to8splits = 0; // # 16-bit accesses split 594ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_64to32pulldown = 0; // # calls to pulldown_to_32 595ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_32to16pulldown = 0; // # calls to pulldown_to_16 596ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_16to8pulldown = 0; // # calls to pulldown_to_8 597ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__vts__tick = 0; // # calls to VTS__tick 598ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__vts__join = 0; // # calls to VTS__join 599ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__vts__cmpLEQ = 0; // # calls to VTS__cmpLEQ 600ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__vts__cmp_structural = 0; // # calls to VTS__cmp_structural 601b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 602b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// # calls to VTS__cmp_structural w/ slow case 603b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic UWord stats__vts__cmp_structural_slow = 0; 604b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 605b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// # calls to VTS__indexAt_SLOW 606b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic UWord stats__vts__indexat_slow = 0; 607b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 608b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// # calls to vts_set__find__or__clone_and_add 609b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic UWord stats__vts_set__focaa = 0; 610b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 611b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// # calls to vts_set__find__or__clone_and_add that lead to an 612b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// allocation 613b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic UWord stats__vts_set__focaa_a = 0; 614ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 615ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 616ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Addr shmem__round_to_SecMap_base ( Addr a ) { 617ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return a & ~(N_SECMAP_ARANGE - 1); 618ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 619ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline UWord shmem__get_SecMap_offset ( Addr a ) { 620ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return a & (N_SECMAP_ARANGE - 1); 621ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 622ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 623ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 624ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*----------------------------------------------------------------*/ 625ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--- map_shmem :: WordFM Addr SecMap ---*/ 626ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--- shadow memory (low level handlers) (shmem__* fns) ---*/ 627ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*----------------------------------------------------------------*/ 628ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 629ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--------------- SecMap allocation --------------- */ 630ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 631ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic HChar* shmem__bigchunk_next = NULL; 632ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic HChar* shmem__bigchunk_end1 = NULL; 633ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 634ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void* shmem__bigchunk_alloc ( SizeT n ) 635ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 636ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown const SizeT sHMEM__BIGCHUNK_SIZE = 4096 * 256 * 4; 637ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(n > 0); 638ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown n = VG_ROUNDUP(n, 16); 639ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(shmem__bigchunk_next <= shmem__bigchunk_end1); 640ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(shmem__bigchunk_end1 - shmem__bigchunk_next 641ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown <= (SSizeT)sHMEM__BIGCHUNK_SIZE); 642ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (shmem__bigchunk_next + n > shmem__bigchunk_end1) { 643ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) 644ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("XXXXX bigchunk: abandoning %d bytes\n", 645ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)(shmem__bigchunk_end1 - shmem__bigchunk_next)); 646ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown shmem__bigchunk_next = VG_(am_shadow_alloc)( sHMEM__BIGCHUNK_SIZE ); 647ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (shmem__bigchunk_next == NULL) 648ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(out_of_memory_NORETURN)( 649ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "helgrind:shmem__bigchunk_alloc", sHMEM__BIGCHUNK_SIZE ); 650ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown shmem__bigchunk_end1 = shmem__bigchunk_next + sHMEM__BIGCHUNK_SIZE; 651ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 652ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(shmem__bigchunk_next); 653ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert( 0 == (((Addr)shmem__bigchunk_next) & (16-1)) ); 654ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(shmem__bigchunk_next + n <= shmem__bigchunk_end1); 655ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown shmem__bigchunk_next += n; 656ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return shmem__bigchunk_next - n; 657ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 658ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 659ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic SecMap* shmem__alloc_SecMap ( void ) 660ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 661ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i, j; 662ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SecMap* sm = shmem__bigchunk_alloc( sizeof(SecMap) ); 663ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)("alloc_SecMap %p\n",sm); 664ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm); 665ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sm->magic = SecMap_MAGIC; 666ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_SECMAP_ZLINES; i++) { 667ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sm->linesZ[i].dict[0] = SVal_NOACCESS; 668ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sm->linesZ[i].dict[1] = SVal_INVALID; 669ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sm->linesZ[i].dict[2] = SVal_INVALID; 670ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sm->linesZ[i].dict[3] = SVal_INVALID; 671ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (j = 0; j < N_LINE_ARANGE/4; j++) 672ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sm->linesZ[i].ix2s[j] = 0; /* all reference dict[0] */ 673ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 674ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sm->linesF = NULL; 675ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sm->linesF_size = 0; 676ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmaps_allocd++; 677ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmap_ga_space_covered += N_SECMAP_ARANGE; 678ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmap_linesZ_allocd += N_SECMAP_ZLINES; 679ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmap_linesZ_bytes += N_SECMAP_ZLINES * sizeof(LineZ); 680ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return sm; 681ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 682ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 683ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef struct { Addr gaKey; SecMap* sm; } SMCacheEnt; 684ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic SMCacheEnt smCache[3] = { {1,NULL}, {1,NULL}, {1,NULL} }; 685ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 686ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic SecMap* shmem__find_SecMap ( Addr ga ) 687ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 688ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SecMap* sm = NULL; 689ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr gaKey = shmem__round_to_SecMap_base(ga); 690ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // Cache 691ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmaps_search++; 692ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(gaKey == smCache[0].gaKey)) 693ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return smCache[0].sm; 694ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(gaKey == smCache[1].gaKey)) { 695ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SMCacheEnt tmp = smCache[0]; 696ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown smCache[0] = smCache[1]; 697ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown smCache[1] = tmp; 698ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return smCache[0].sm; 699ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 700ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (gaKey == smCache[2].gaKey) { 701ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SMCacheEnt tmp = smCache[1]; 702ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown smCache[1] = smCache[2]; 703ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown smCache[2] = tmp; 704ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return smCache[1].sm; 705ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 706ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // end Cache 707ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmaps_search_slow++; 708ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (VG_(lookupFM)( map_shmem, 709ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown NULL/*keyP*/, (UWord*)&sm, (UWord)gaKey )) { 710ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm != NULL); 711ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown smCache[2] = smCache[1]; 712ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown smCache[1] = smCache[0]; 713ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown smCache[0].gaKey = gaKey; 714ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown smCache[0].sm = sm; 715ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 716ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm == NULL); 717ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 718ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return sm; 719ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 720ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 721ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic SecMap* shmem__find_or_alloc_SecMap ( Addr ga ) 722ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 723ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SecMap* sm = shmem__find_SecMap ( ga ); 724ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(sm)) { 725ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return sm; 726ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 727ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* create a new one */ 728ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr gaKey = shmem__round_to_SecMap_base(ga); 729ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sm = shmem__alloc_SecMap(); 730ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm); 731ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(addToFM)( map_shmem, (UWord)gaKey, (UWord)sm ); 732ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return sm; 733ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 734ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 735ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 736ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 737ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* ------------ LineF and LineZ related ------------ */ 738ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 739ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void rcinc_LineF ( LineF* lineF ) { 740ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i; 741ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(lineF->inUse); 742ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_LINE_ARANGE; i++) 743ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcinc(lineF->w64s[i]); 744ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 745ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 746ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void rcdec_LineF ( LineF* lineF ) { 747ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i; 748ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(lineF->inUse); 749ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_LINE_ARANGE; i++) 750ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcdec(lineF->w64s[i]); 751ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 752ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 753ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void rcinc_LineZ ( LineZ* lineZ ) { 754ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(lineZ->dict[0] != SVal_INVALID); 755ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcinc(lineZ->dict[0]); 756ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lineZ->dict[1] != SVal_INVALID) rcinc(lineZ->dict[1]); 757ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lineZ->dict[2] != SVal_INVALID) rcinc(lineZ->dict[2]); 758ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lineZ->dict[3] != SVal_INVALID) rcinc(lineZ->dict[3]); 759ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 760ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 761ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void rcdec_LineZ ( LineZ* lineZ ) { 762ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(lineZ->dict[0] != SVal_INVALID); 763ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcdec(lineZ->dict[0]); 764ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lineZ->dict[1] != SVal_INVALID) rcdec(lineZ->dict[1]); 765ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lineZ->dict[2] != SVal_INVALID) rcdec(lineZ->dict[2]); 766ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lineZ->dict[3] != SVal_INVALID) rcdec(lineZ->dict[3]); 767ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 768ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 769ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browninline 770ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void write_twobit_array ( UChar* arr, UWord ix, UWord b2 ) { 771ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word bix, shft, mask, prep; 772ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ix >= 0); 773ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown bix = ix >> 2; 774ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown shft = 2 * (ix & 3); /* 0, 2, 4 or 6 */ 775ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mask = 3 << shft; 776ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown prep = b2 << shft; 777ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown arr[bix] = (arr[bix] & ~mask) | prep; 778ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 779ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 780ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browninline 781ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord read_twobit_array ( UChar* arr, UWord ix ) { 782ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word bix, shft; 783ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ix >= 0); 784ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown bix = ix >> 2; 785ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown shft = 2 * (ix & 3); /* 0, 2, 4 or 6 */ 786ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return (arr[bix] >> shft) & 3; 787ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 788ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 789ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Given address 'tag', find either the Z or F line containing relevant 790ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown data, so it can be read into the cache. 791ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 792ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void find_ZF_for_reading ( /*OUT*/LineZ** zp, 793ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /*OUT*/LineF** fp, Addr tag ) { 794ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineZ* lineZ; 795ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineF* lineF; 796ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord zix; 797ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SecMap* sm = shmem__find_or_alloc_SecMap(tag); 798ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord smoff = shmem__get_SecMap_offset(tag); 799ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* since smoff is derived from a valid tag, it should be 800ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cacheline-aligned. */ 801ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0 == (smoff & (N_LINE_ARANGE - 1))); 802ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zix = smoff >> N_LINE_BITS; 803ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(zix < N_SECMAP_ZLINES); 804ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineZ = &sm->linesZ[zix]; 805ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineF = NULL; 806ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lineZ->dict[0] == SVal_INVALID) { 807ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt fix = (UInt)lineZ->dict[1]; 808ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm->linesF); 809ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm->linesF_size > 0); 810ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(fix >= 0 && fix < sm->linesF_size); 811ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineF = &sm->linesF[fix]; 812ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(lineF->inUse); 813ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineZ = NULL; 814ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 815ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *zp = lineZ; 816ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *fp = lineF; 817ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 818ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 819ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Given address 'tag', return the relevant SecMap and the index of 820ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown the LineZ within it, in the expectation that the line is to be 821ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown overwritten. Regardless of whether 'tag' is currently associated 822ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown with a Z or F representation, to rcdec on the current 823ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown representation, in recognition of the fact that the contents are 824ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown just about to be overwritten. */ 825ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __attribute__((noinline)) 826ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid find_Z_for_writing ( /*OUT*/SecMap** smp, 827ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /*OUT*/Word* zixp, 828ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr tag ) { 829ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineZ* lineZ; 830ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineF* lineF; 831ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord zix; 832ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SecMap* sm = shmem__find_or_alloc_SecMap(tag); 833ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord smoff = shmem__get_SecMap_offset(tag); 834ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* since smoff is derived from a valid tag, it should be 835ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cacheline-aligned. */ 836ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0 == (smoff & (N_LINE_ARANGE - 1))); 837ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zix = smoff >> N_LINE_BITS; 838ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(zix < N_SECMAP_ZLINES); 839ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineZ = &sm->linesZ[zix]; 840ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineF = NULL; 841ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* re RCs, we are freeing up this LineZ/LineF so that new data can 842ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown be parked in it. Hence have to rcdec it accordingly. */ 843ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* If lineZ has an associated lineF, free it up. */ 844ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lineZ->dict[0] == SVal_INVALID) { 845ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt fix = (UInt)lineZ->dict[1]; 846ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm->linesF); 847ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm->linesF_size > 0); 848ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(fix >= 0 && fix < sm->linesF_size); 849ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineF = &sm->linesF[fix]; 850ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(lineF->inUse); 851ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcdec_LineF(lineF); 852ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineF->inUse = False; 853ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 854ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcdec_LineZ(lineZ); 855ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 856ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *smp = sm; 857ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *zixp = zix; 858ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 859ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 860ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __attribute__((noinline)) 861ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid alloc_F_for_writing ( /*MOD*/SecMap* sm, /*OUT*/Word* fixp ) { 862ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt i, new_size; 863ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineF* nyu; 864ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 865ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (sm->linesF) { 866ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm->linesF_size > 0); 867ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 868ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm->linesF_size == 0); 869ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 870ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 871ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (sm->linesF) { 872ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < sm->linesF_size; i++) { 873ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!sm->linesF[i].inUse) { 874ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *fixp = (Word)i; 875ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 876ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 877ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 878ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 879ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 880ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* No free F line found. Expand existing array and try again. */ 881ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown new_size = sm->linesF_size==0 ? 1 : 2 * sm->linesF_size; 882ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nyu = HG_(zalloc)( "libhb.aFfw.1 (LineF storage)", 883ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown new_size * sizeof(LineF) ); 884ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(nyu); 885ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 886ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmap_linesF_allocd += (new_size - sm->linesF_size); 887ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmap_linesF_bytes += (new_size - sm->linesF_size) 888ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown * sizeof(LineF); 889ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 890ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) 891ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("SM %p: expand F array from %d to %d\n", 892ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sm, (Int)sm->linesF_size, new_size); 893ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 894ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < new_size; i++) 895ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nyu[i].inUse = False; 896ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 897ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (sm->linesF) { 898ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < sm->linesF_size; i++) { 899ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm->linesF[i].inUse); 900ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nyu[i] = sm->linesF[i]; 901ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 902ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(memset)(sm->linesF, 0, sm->linesF_size * sizeof(LineF) ); 903ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free)(sm->linesF); 904ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 905ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 906ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sm->linesF = nyu; 907ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sm->linesF_size = new_size; 908ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 909ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < sm->linesF_size; i++) { 910ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!sm->linesF[i].inUse) { 911ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *fixp = (Word)i; 912ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 913ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 914ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 915ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 916ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /*NOTREACHED*/ 917ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 918ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 919ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 920ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 921ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* ------------ CacheLine and implicit-tree related ------------ */ 922ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 923ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown__attribute__((unused)) 924ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void pp_CacheLine ( CacheLine* cl ) { 925ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i; 926ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!cl) { 927ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","pp_CacheLine(NULL)\n"); 928ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 929ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 930ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_LINE_TREES; i++) 931ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" descr: %04lx\n", (UWord)cl->descrs[i]); 932ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_LINE_ARANGE; i++) 933ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" sval: %08lx\n", (UWord)cl->svals[i]); 934ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 935ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 936ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UChar descr_to_validbits ( UShort descr ) 937ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 938ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* a.k.a Party Time for gcc's constant folder */ 939ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# define DESCR(b8_7, b8_6, b8_5, b8_4, b8_3, b8_2, b8_1, b8_0, \ 940ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b16_3, b32_1, b16_2, b64, b16_1, b32_0, b16_0) \ 941ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ( (UShort) ( ( (b8_7) << 14) | ( (b8_6) << 13) | \ 942ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ( (b8_5) << 12) | ( (b8_4) << 11) | \ 943ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ( (b8_3) << 10) | ( (b8_2) << 9) | \ 944ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ( (b8_1) << 8) | ( (b8_0) << 7) | \ 945ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ( (b16_3) << 6) | ( (b32_1) << 5) | \ 946ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ( (b16_2) << 4) | ( (b64) << 3) | \ 947ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ( (b16_1) << 2) | ( (b32_0) << 1) | \ 948ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ( (b16_0) << 0) ) ) 949ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 950ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# define BYTE(bit7, bit6, bit5, bit4, bit3, bit2, bit1, bit0) \ 951ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ( (UChar) ( ( (bit7) << 7) | ( (bit6) << 6) | \ 952ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ( (bit5) << 5) | ( (bit4) << 4) | \ 953ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ( (bit3) << 3) | ( (bit2) << 2) | \ 954ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ( (bit1) << 1) | ( (bit0) << 0) ) ) 955ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 956ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* these should all get folded out at compile time */ 957ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(DESCR(1,0,0,0,0,0,0,0, 0,0,0, 0, 0,0,0) == TREE_DESCR_8_7); 958ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(DESCR(0,0,0,0,0,0,0,1, 0,0,0, 0, 0,0,0) == TREE_DESCR_8_0); 959ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(DESCR(0,0,0,0,0,0,0,0, 1,0,0, 0, 0,0,0) == TREE_DESCR_16_3); 960ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(DESCR(0,0,0,0,0,0,0,0, 0,1,0, 0, 0,0,0) == TREE_DESCR_32_1); 961ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(DESCR(0,0,0,0,0,0,0,0, 0,0,1, 0, 0,0,0) == TREE_DESCR_16_2); 962ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(DESCR(0,0,0,0,0,0,0,0, 0,0,0, 1, 0,0,0) == TREE_DESCR_64); 963ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(DESCR(0,0,0,0,0,0,0,0, 0,0,0, 0, 1,0,0) == TREE_DESCR_16_1); 964ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(DESCR(0,0,0,0,0,0,0,0, 0,0,0, 0, 0,1,0) == TREE_DESCR_32_0); 965ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(DESCR(0,0,0,0,0,0,0,0, 0,0,0, 0, 0,0,1) == TREE_DESCR_16_0); 966ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 967ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown switch (descr) { 968ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* 969ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown +--------------------------------- TREE_DESCR_8_7 970ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | +------------------- TREE_DESCR_8_0 971ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | | +---------------- TREE_DESCR_16_3 972ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | | | +-------------- TREE_DESCR_32_1 973ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | | | | +------------ TREE_DESCR_16_2 974ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | | | | | +--------- TREE_DESCR_64 975ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | | | | | | +------ TREE_DESCR_16_1 976ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | | | | | | | +---- TREE_DESCR_32_0 977ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | | | | | | | | +-- TREE_DESCR_16_0 978ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | | | | | | | | | 979ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | | | | | | | | | GRANULARITY, 7 -> 0 */ 980ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(1,1,1,1,1,1,1,1, 0,0,0, 0, 0,0,0): /* 8 8 8 8 8 8 8 8 */ 981ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(1,1,1,1,1,1,1,1); 982ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(1,1,0,0,1,1,1,1, 0,0,1, 0, 0,0,0): /* 8 8 16 8 8 8 8 */ 983ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(1,1,0,1,1,1,1,1); 984ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,1,1,1,1,1,1, 1,0,0, 0, 0,0,0): /* 16 8 8 8 8 8 8 */ 985ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,1,1,1,1,1,1,1); 986ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,0,0,1,1,1,1, 1,0,1, 0, 0,0,0): /* 16 16 8 8 8 8 */ 987ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,1,0,1,1,1,1,1); 988ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 989ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(1,1,1,1,1,1,0,0, 0,0,0, 0, 0,0,1): /* 8 8 8 8 8 8 16 */ 990ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(1,1,1,1,1,1,0,1); 991ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(1,1,0,0,1,1,0,0, 0,0,1, 0, 0,0,1): /* 8 8 16 8 8 16 */ 992ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(1,1,0,1,1,1,0,1); 993ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,1,1,1,1,0,0, 1,0,0, 0, 0,0,1): /* 16 8 8 8 8 16 */ 994ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,1,1,1,1,1,0,1); 995ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,0,0,1,1,0,0, 1,0,1, 0, 0,0,1): /* 16 16 8 8 16 */ 996ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,1,0,1,1,1,0,1); 997ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 998ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(1,1,1,1,0,0,1,1, 0,0,0, 0, 1,0,0): /* 8 8 8 8 16 8 8 */ 999ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(1,1,1,1,0,1,1,1); 1000ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(1,1,0,0,0,0,1,1, 0,0,1, 0, 1,0,0): /* 8 8 16 16 8 8 */ 1001ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(1,1,0,1,0,1,1,1); 1002ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,1,1,0,0,1,1, 1,0,0, 0, 1,0,0): /* 16 8 8 16 8 8 */ 1003ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,1,1,1,0,1,1,1); 1004ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,0,0,0,0,1,1, 1,0,1, 0, 1,0,0): /* 16 16 16 8 8 */ 1005ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,1,0,1,0,1,1,1); 1006ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1007ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(1,1,1,1,0,0,0,0, 0,0,0, 0, 1,0,1): /* 8 8 8 8 16 16 */ 1008ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(1,1,1,1,0,1,0,1); 1009ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(1,1,0,0,0,0,0,0, 0,0,1, 0, 1,0,1): /* 8 8 16 16 16 */ 1010ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(1,1,0,1,0,1,0,1); 1011ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,1,1,0,0,0,0, 1,0,0, 0, 1,0,1): /* 16 8 8 16 16 */ 1012ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,1,1,1,0,1,0,1); 1013ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,0,0,0,0,0,0, 1,0,1, 0, 1,0,1): /* 16 16 16 16 */ 1014ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,1,0,1,0,1,0,1); 1015ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1016ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,0,0,1,1,1,1, 0,1,0, 0, 0,0,0): /* 32 8 8 8 8 */ 1017ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,0,0,1,1,1,1,1); 1018ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,0,0,1,1,0,0, 0,1,0, 0, 0,0,1): /* 32 8 8 16 */ 1019ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,0,0,1,1,1,0,1); 1020ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,0,0,0,0,1,1, 0,1,0, 0, 1,0,0): /* 32 16 8 8 */ 1021ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,0,0,1,0,1,1,1); 1022ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,0,0,0,0,0,0, 0,1,0, 0, 1,0,1): /* 32 16 16 */ 1023ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,0,0,1,0,1,0,1); 1024ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1025ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(1,1,1,1,0,0,0,0, 0,0,0, 0, 0,1,0): /* 8 8 8 8 32 */ 1026ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(1,1,1,1,0,0,0,1); 1027ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(1,1,0,0,0,0,0,0, 0,0,1, 0, 0,1,0): /* 8 8 16 32 */ 1028ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(1,1,0,1,0,0,0,1); 1029ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,1,1,0,0,0,0, 1,0,0, 0, 0,1,0): /* 16 8 8 32 */ 1030ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,1,1,1,0,0,0,1); 1031ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,0,0,0,0,0,0, 1,0,1, 0, 0,1,0): /* 16 16 32 */ 1032ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,1,0,1,0,0,0,1); 1033ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1034ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,0,0,0,0,0,0, 0,1,0, 0, 0,1,0): /* 32 32 */ 1035ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,0,0,1,0,0,0,1); 1036ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1037ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,0,0,0,0,0,0, 0,0,0, 1, 0,0,0): /* 64 */ 1038ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,0,0,0,0,0,0,1); 1039ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1040ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown default: return BYTE(0,0,0,0,0,0,0,0); 1041ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* INVALID - any valid descr produces at least one 1042ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown valid bit in tree[0..7]*/ 1043ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1044ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* NOTREACHED*/ 1045ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 1046ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1047ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# undef DESCR 1048ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# undef BYTE 1049ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1050ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1051ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown__attribute__((unused)) 1052ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Bool is_sane_Descr ( UShort descr ) { 1053ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return descr_to_validbits(descr) != 0; 1054ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1055ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1056ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void sprintf_Descr ( /*OUT*/HChar* dst, UShort descr ) { 1057ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(sprintf)(dst, 1058ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "%d%d%d%d%d%d%d%d %d%d%d %d %d%d%d", 1059ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_8_7) ? 1 : 0), 1060ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_8_6) ? 1 : 0), 1061ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_8_5) ? 1 : 0), 1062ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_8_4) ? 1 : 0), 1063ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_8_3) ? 1 : 0), 1064ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_8_2) ? 1 : 0), 1065ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_8_1) ? 1 : 0), 1066ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_8_0) ? 1 : 0), 1067ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_16_3) ? 1 : 0), 1068ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_32_1) ? 1 : 0), 1069ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_16_2) ? 1 : 0), 1070ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_64) ? 1 : 0), 1071ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_16_1) ? 1 : 0), 1072ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_32_0) ? 1 : 0), 1073ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_16_0) ? 1 : 0) 1074ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ); 1075ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1076ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void sprintf_Byte ( /*OUT*/HChar* dst, UChar byte ) { 1077ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(sprintf)(dst, "%d%d%d%d%d%d%d%d", 1078ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((byte & 128) ? 1 : 0), 1079ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((byte & 64) ? 1 : 0), 1080ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((byte & 32) ? 1 : 0), 1081ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((byte & 16) ? 1 : 0), 1082ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((byte & 8) ? 1 : 0), 1083ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((byte & 4) ? 1 : 0), 1084ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((byte & 2) ? 1 : 0), 1085ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((byte & 1) ? 1 : 0) 1086ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ); 1087ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1088ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1089ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Bool is_sane_Descr_and_Tree ( UShort descr, SVal* tree ) { 1090ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i; 1091ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UChar validbits = descr_to_validbits(descr); 1092ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HChar buf[128], buf2[128]; 1093ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (validbits == 0) 1094ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto bad; 1095ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < 8; i++) { 1096ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (validbits & (1<<i)) { 1097ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tree[i] == SVal_INVALID) 1098ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto bad; 1099ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 1100ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tree[i] != SVal_INVALID) 1101ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto bad; 1102ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1103ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1104ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return True; 1105ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown bad: 1106ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sprintf_Descr( buf, descr ); 1107ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sprintf_Byte( buf2, validbits ); 1108ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","is_sane_Descr_and_Tree: bad tree {\n"); 1109ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" validbits 0x%02lx %s\n", (UWord)validbits, buf2); 1110ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" descr 0x%04lx %s\n", (UWord)descr, buf); 1111ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < 8; i++) 1112ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" [%ld] 0x%016llx\n", i, tree[i]); 1113ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","}\n"); 1114ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0; 1115ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1116ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1117ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Bool is_sane_CacheLine ( CacheLine* cl ) 1118ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1119ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word tno, cloff; 1120ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1121ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!cl) goto bad; 1122ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1123ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (tno = 0, cloff = 0; tno < N_LINE_TREES; tno++, cloff += 8) { 1124ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr = cl->descrs[tno]; 1125ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal* tree = &cl->svals[cloff]; 1126ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!is_sane_Descr_and_Tree(descr, tree)) 1127ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto bad; 1128ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1129ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(cloff == N_LINE_ARANGE); 1130ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return True; 1131ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown bad: 1132ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown pp_CacheLine(cl); 1133ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 1134ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1135ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1136ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UShort normalise_tree ( /*MOD*/SVal* tree ) 1137ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1138ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr; 1139ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* pre: incoming tree[0..7] does not have any invalid shvals, in 1140ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown particular no zeroes. */ 1141ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(tree[7] == SVal_INVALID || tree[6] == SVal_INVALID 1142ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown || tree[5] == SVal_INVALID || tree[4] == SVal_INVALID 1143ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown || tree[3] == SVal_INVALID || tree[2] == SVal_INVALID 1144ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown || tree[1] == SVal_INVALID || tree[0] == SVal_INVALID)) 1145ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 1146ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1147ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = TREE_DESCR_8_7 | TREE_DESCR_8_6 | TREE_DESCR_8_5 1148ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | TREE_DESCR_8_4 | TREE_DESCR_8_3 | TREE_DESCR_8_2 1149ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | TREE_DESCR_8_1 | TREE_DESCR_8_0; 1150ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* build 16-bit layer */ 1151ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tree[1] == tree[0]) { 1152ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[1] = SVal_INVALID; 1153ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~(TREE_DESCR_8_1 | TREE_DESCR_8_0); 1154ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= TREE_DESCR_16_0; 1155ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1156ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tree[3] == tree[2]) { 1157ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[3] = SVal_INVALID; 1158ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~(TREE_DESCR_8_3 | TREE_DESCR_8_2); 1159ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= TREE_DESCR_16_1; 1160ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1161ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tree[5] == tree[4]) { 1162ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[5] = SVal_INVALID; 1163ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~(TREE_DESCR_8_5 | TREE_DESCR_8_4); 1164ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= TREE_DESCR_16_2; 1165ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1166ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tree[7] == tree[6]) { 1167ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[7] = SVal_INVALID; 1168ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~(TREE_DESCR_8_7 | TREE_DESCR_8_6); 1169ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= TREE_DESCR_16_3; 1170ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1171ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* build 32-bit layer */ 1172ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tree[2] == tree[0] 1173ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown && (descr & TREE_DESCR_16_1) && (descr & TREE_DESCR_16_0)) { 1174ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[2] = SVal_INVALID; /* [3,1] must already be SVal_INVALID */ 1175ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~(TREE_DESCR_16_1 | TREE_DESCR_16_0); 1176ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= TREE_DESCR_32_0; 1177ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1178ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tree[6] == tree[4] 1179ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown && (descr & TREE_DESCR_16_3) && (descr & TREE_DESCR_16_2)) { 1180ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[6] = SVal_INVALID; /* [7,5] must already be SVal_INVALID */ 1181ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~(TREE_DESCR_16_3 | TREE_DESCR_16_2); 1182ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= TREE_DESCR_32_1; 1183ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1184ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* build 64-bit layer */ 1185ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tree[4] == tree[0] 1186ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown && (descr & TREE_DESCR_32_1) && (descr & TREE_DESCR_32_0)) { 1187ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[4] = SVal_INVALID; /* [7,6,5,3,2,1] must already be SVal_INVALID */ 1188ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~(TREE_DESCR_32_1 | TREE_DESCR_32_0); 1189ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= TREE_DESCR_64; 1190ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1191ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return descr; 1192ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1193ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1194ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* This takes a cacheline where all the data is at the leaves 1195ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (w8[..]) and builds a correctly normalised tree. */ 1196ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void normalise_CacheLine ( /*MOD*/CacheLine* cl ) 1197ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1198ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word tno, cloff; 1199ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (tno = 0, cloff = 0; tno < N_LINE_TREES; tno++, cloff += 8) { 1200ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal* tree = &cl->svals[cloff]; 1201ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = normalise_tree( tree ); 1202ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1203ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(cloff == N_LINE_ARANGE); 1204ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 1205ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */ 1206ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_normalises++; 1207ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1208ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1209ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1210ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef struct { UChar count; SVal sval; } CountedSVal; 1211ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1212ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 1213ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid sequentialise_CacheLine ( /*OUT*/CountedSVal* dst, 1214ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /*OUT*/Word* dstUsedP, 1215ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word nDst, CacheLine* src ) 1216ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1217ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word tno, cloff, dstUsed; 1218ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1219ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(nDst == N_LINE_ARANGE); 1220ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown dstUsed = 0; 1221ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1222ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (tno = 0, cloff = 0; tno < N_LINE_TREES; tno++, cloff += 8) { 1223ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr = src->descrs[tno]; 1224ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal* tree = &src->svals[cloff]; 1225ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1226ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* sequentialise the tree described by (descr,tree). */ 1227ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# define PUT(_n,_v) \ 1228ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown do { dst[dstUsed ].count = (_n); \ 1229ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown dst[dstUsed++].sval = (_v); \ 1230ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } while (0) 1231ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1232ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* byte 0 */ 1233ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_64) PUT(8, tree[0]); else 1234ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_32_0) PUT(4, tree[0]); else 1235ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_16_0) PUT(2, tree[0]); else 1236ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_8_0) PUT(1, tree[0]); 1237ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* byte 1 */ 1238ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_8_1) PUT(1, tree[1]); 1239ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* byte 2 */ 1240ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_16_1) PUT(2, tree[2]); else 1241ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_8_2) PUT(1, tree[2]); 1242ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* byte 3 */ 1243ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_8_3) PUT(1, tree[3]); 1244ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* byte 4 */ 1245ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_32_1) PUT(4, tree[4]); else 1246ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_16_2) PUT(2, tree[4]); else 1247ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_8_4) PUT(1, tree[4]); 1248ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* byte 5 */ 1249ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_8_5) PUT(1, tree[5]); 1250ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* byte 6 */ 1251ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_16_3) PUT(2, tree[6]); else 1252ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_8_6) PUT(1, tree[6]); 1253ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* byte 7 */ 1254ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_8_7) PUT(1, tree[7]); 1255ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1256ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# undef PUT 1257ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* END sequentialise the tree described by (descr,tree). */ 1258ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1259ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1260ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(cloff == N_LINE_ARANGE); 1261ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(dstUsed <= nDst); 1262ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1263ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *dstUsedP = dstUsed; 1264ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1265ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1266ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Write the cacheline 'wix' to backing store. Where it ends up 1267ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown is determined by its tag field. */ 1268ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __attribute__((noinline)) void cacheline_wback ( UWord wix ) 1269ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1270ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i, j, k, m; 1271ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr tag; 1272ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SecMap* sm; 1273ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 1274ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineZ* lineZ; 1275ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineF* lineF; 1276ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word zix, fix, csvalsUsed; 1277ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CountedSVal csvals[N_LINE_ARANGE]; 1278ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal sv; 1279ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1280ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) 1281ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("scache wback line %d\n", (Int)wix); 1282ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1283ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(wix >= 0 && wix < N_WAY_NENT); 1284ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1285ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tag = cache_shmem.tags0[wix]; 1286ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = &cache_shmem.lyns0[wix]; 1287ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1288ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* The cache line may have been invalidated; if so, ignore it. */ 1289ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!is_valid_scache_tag(tag)) 1290ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 1291ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1292ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Where are we going to put it? */ 1293ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sm = NULL; 1294ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineZ = NULL; 1295ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineF = NULL; 1296ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zix = fix = -1; 1297ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1298ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* find the Z line to write in and rcdec it or the associated F 1299ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line. */ 1300ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown find_Z_for_writing( &sm, &zix, tag ); 1301ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1302ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm); 1303ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(zix >= 0 && zix < N_SECMAP_ZLINES); 1304ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineZ = &sm->linesZ[zix]; 1305ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1306ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Generate the data to be stored */ 1307ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 1308ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */ 1309ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1310ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown csvalsUsed = -1; 1311ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sequentialise_CacheLine( csvals, &csvalsUsed, 1312ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown N_LINE_ARANGE, cl ); 1313ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(csvalsUsed >= 1 && csvalsUsed <= N_LINE_ARANGE); 1314ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)("%lu ", csvalsUsed); 1315ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1316ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineZ->dict[0] = lineZ->dict[1] 1317ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown = lineZ->dict[2] = lineZ->dict[3] = SVal_INVALID; 1318ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1319ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* i indexes actual shadow values, k is cursor in csvals */ 1320ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown i = 0; 1321ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (k = 0; k < csvalsUsed; k++) { 1322ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1323ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sv = csvals[k].sval; 1324ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 1325ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(csvals[k].count >= 1 && csvals[k].count <= 8); 1326ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* do we already have it? */ 1327ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (sv == lineZ->dict[0]) { j = 0; goto dict_ok; } 1328ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (sv == lineZ->dict[1]) { j = 1; goto dict_ok; } 1329ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (sv == lineZ->dict[2]) { j = 2; goto dict_ok; } 1330ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (sv == lineZ->dict[3]) { j = 3; goto dict_ok; } 1331ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* no. look for a free slot. */ 1332ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 1333ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sv != SVal_INVALID); 1334ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lineZ->dict[0] 1335ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown == SVal_INVALID) { lineZ->dict[0] = sv; j = 0; goto dict_ok; } 1336ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lineZ->dict[1] 1337ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown == SVal_INVALID) { lineZ->dict[1] = sv; j = 1; goto dict_ok; } 1338ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lineZ->dict[2] 1339ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown == SVal_INVALID) { lineZ->dict[2] = sv; j = 2; goto dict_ok; } 1340ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lineZ->dict[3] 1341ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown == SVal_INVALID) { lineZ->dict[3] = sv; j = 3; goto dict_ok; } 1342ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; /* we'll have to use the f rep */ 1343ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown dict_ok: 1344ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown m = csvals[k].count; 1345ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (m == 8) { 1346ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+0, j ); 1347ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+1, j ); 1348ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+2, j ); 1349ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+3, j ); 1350ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+4, j ); 1351ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+5, j ); 1352ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+6, j ); 1353ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+7, j ); 1354ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown i += 8; 1355ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1356ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown else if (m == 4) { 1357ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+0, j ); 1358ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+1, j ); 1359ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+2, j ); 1360ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+3, j ); 1361ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown i += 4; 1362ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1363ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown else if (m == 1) { 1364ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+0, j ); 1365ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown i += 1; 1366ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1367ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown else if (m == 2) { 1368ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+0, j ); 1369ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+1, j ); 1370ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown i += 2; 1371ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1372ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown else { 1373ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); /* 8 4 2 or 1 are the only legitimate values for m */ 1374ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1375ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1376ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1377ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1378ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(i == N_LINE_ARANGE)) { 1379ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Construction of the compressed representation was 1380ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown successful. */ 1381ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcinc_LineZ(lineZ); 1382ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_Z_wbacks++; 1383ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 1384ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Cannot use the compressed(z) representation. Use the full(f) 1385ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rep instead. */ 1386ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(i >= 0 && i < N_LINE_ARANGE); 1387ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown alloc_F_for_writing( sm, &fix ); 1388ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm->linesF); 1389ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm->linesF_size > 0); 1390ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(fix >= 0 && fix < (Word)sm->linesF_size); 1391ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineF = &sm->linesF[fix]; 1392ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(!lineF->inUse); 1393ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineZ->dict[0] = lineZ->dict[2] = lineZ->dict[3] = SVal_INVALID; 1394ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineZ->dict[1] = (SVal)fix; 1395ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineF->inUse = True; 1396ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown i = 0; 1397ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (k = 0; k < csvalsUsed; k++) { 1398ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 1399ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(csvals[k].count >= 1 && csvals[k].count <= 8); 1400ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sv = csvals[k].sval; 1401ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 1402ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sv != SVal_INVALID); 1403ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (m = csvals[k].count; m > 0; m--) { 1404ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineF->w64s[i] = sv; 1405ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown i++; 1406ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1407ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1408ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(i == N_LINE_ARANGE); 1409ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcinc_LineF(lineF); 1410ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_F_wbacks++; 1411ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1412ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1413ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1414ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Fetch the cacheline 'wix' from the backing store. The tag 1415ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown associated with 'wix' is assumed to have already been filled in; 1416ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown hence that is used to determine where in the backing store to read 1417ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown from. */ 1418ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __attribute__((noinline)) void cacheline_fetch ( UWord wix ) 1419ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1420ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i; 1421ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr tag; 1422ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 1423ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineZ* lineZ; 1424ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineF* lineF; 1425ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1426ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) 1427ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("scache fetch line %d\n", (Int)wix); 1428ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1429ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(wix >= 0 && wix < N_WAY_NENT); 1430ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1431ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tag = cache_shmem.tags0[wix]; 1432ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = &cache_shmem.lyns0[wix]; 1433ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1434ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* reject nonsense requests */ 1435ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_valid_scache_tag(tag)); 1436ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1437ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineZ = NULL; 1438ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineF = NULL; 1439ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown find_ZF_for_reading( &lineZ, &lineF, tag ); 1440ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert( (lineZ && !lineF) || (!lineZ && lineF) ); 1441ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1442ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* expand the data into the bottom layer of the tree, then get 1443ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cacheline_normalise to build the descriptor array. */ 1444ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lineF) { 1445ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(lineF->inUse); 1446ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_LINE_ARANGE; i++) { 1447ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[i] = lineF->w64s[i]; 1448ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1449ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_F_fetches++; 1450ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 1451ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_LINE_ARANGE; i++) { 1452ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal sv; 1453ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord ix = read_twobit_array( lineZ->ix2s, i ); 1454ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* correct, but expensive: tl_assert(ix >= 0 && ix <= 3); */ 1455ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sv = lineZ->dict[ix]; 1456ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sv != SVal_INVALID); 1457ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[i] = sv; 1458ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1459ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_Z_fetches++; 1460ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1461ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown normalise_CacheLine( cl ); 1462ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1463ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1464ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void shmem__invalidate_scache ( void ) { 1465ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word wix; 1466ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)("%s","scache inval\n"); 1467ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(!is_valid_scache_tag(1)); 1468ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (wix = 0; wix < N_WAY_NENT; wix++) { 1469ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cache_shmem.tags0[wix] = 1/*INVALID*/; 1470ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1471ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_invals++; 1472ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1473ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1474ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void shmem__flush_and_invalidate_scache ( void ) { 1475ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word wix; 1476ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr tag; 1477ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)("%s","scache flush and invalidate\n"); 1478ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(!is_valid_scache_tag(1)); 1479ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (wix = 0; wix < N_WAY_NENT; wix++) { 1480ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tag = cache_shmem.tags0[wix]; 1481ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tag == 1/*INVALID*/) { 1482ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* already invalid; nothing to do */ 1483ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 1484ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_valid_scache_tag(tag)); 1485ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cacheline_wback( wix ); 1486ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1487ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cache_shmem.tags0[wix] = 1/*INVALID*/; 1488ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1489ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_flushes++; 1490ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_invals++; 1491ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1492ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1493ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1494ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool aligned16 ( Addr a ) { 1495ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0 == (a & 1); 1496ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1497ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool aligned32 ( Addr a ) { 1498ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0 == (a & 3); 1499ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1500ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool aligned64 ( Addr a ) { 1501ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0 == (a & 7); 1502ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1503ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline UWord get_cacheline_offset ( Addr a ) { 1504ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return (UWord)(a & (N_LINE_ARANGE - 1)); 1505ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1506ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Addr cacheline_ROUNDUP ( Addr a ) { 1507ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ROUNDUP(a, N_LINE_ARANGE); 1508ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1509ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Addr cacheline_ROUNDDN ( Addr a ) { 1510ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ROUNDDN(a, N_LINE_ARANGE); 1511ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1512ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline UWord get_treeno ( Addr a ) { 1513ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return get_cacheline_offset(a) >> 3; 1514ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1515ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline UWord get_tree_offset ( Addr a ) { 1516ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return a & 7; 1517ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1518ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1519ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __attribute__((noinline)) 1520ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* get_cacheline_MISS ( Addr a ); /* fwds */ 1521ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline CacheLine* get_cacheline ( Addr a ) 1522ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1523ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* tag is 'a' with the in-line offset masked out, 1524ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown eg a[31]..a[4] 0000 */ 1525ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr tag = a & ~(N_LINE_ARANGE - 1); 1526ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord wix = (a >> N_LINE_BITS) & (N_WAY_NENT - 1); 1527ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_totrefs++; 1528ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(tag == cache_shmem.tags0[wix])) { 1529ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return &cache_shmem.lyns0[wix]; 1530ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 1531ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return get_cacheline_MISS( a ); 1532ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1533ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1534ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1535ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __attribute__((noinline)) 1536ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* get_cacheline_MISS ( Addr a ) 1537ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1538ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* tag is 'a' with the in-line offset masked out, 1539ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown eg a[31]..a[4] 0000 */ 1540ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1541ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 1542ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr* tag_old_p; 1543ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr tag = a & ~(N_LINE_ARANGE - 1); 1544ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord wix = (a >> N_LINE_BITS) & (N_WAY_NENT - 1); 1545ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1546ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(tag != cache_shmem.tags0[wix]); 1547ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1548ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Dump the old line into the backing store. */ 1549ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_totmisses++; 1550ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1551ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = &cache_shmem.lyns0[wix]; 1552ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tag_old_p = &cache_shmem.tags0[wix]; 1553ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1554ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (is_valid_scache_tag( *tag_old_p )) { 1555ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* EXPENSIVE and REDUNDANT: callee does it */ 1556ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 1557ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */ 1558ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cacheline_wback( wix ); 1559ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1560ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* and reload the new one */ 1561ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *tag_old_p = tag; 1562ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cacheline_fetch( wix ); 1563ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 1564ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */ 1565ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return cl; 1566ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1567ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1568ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UShort pulldown_to_32 ( /*MOD*/SVal* tree, UWord toff, UShort descr ) { 1569ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_64to32pulldown++; 1570ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown switch (toff) { 1571ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 0: case 4: 1572ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(descr & TREE_DESCR_64); 1573ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[4] = tree[0]; 1574ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~TREE_DESCR_64; 1575ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= (TREE_DESCR_32_1 | TREE_DESCR_32_0); 1576ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1577ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown default: 1578ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 1579ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1580ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return descr; 1581ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1582ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1583ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UShort pulldown_to_16 ( /*MOD*/SVal* tree, UWord toff, UShort descr ) { 1584ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_32to16pulldown++; 1585ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown switch (toff) { 1586ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 0: case 2: 1587ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!(descr & TREE_DESCR_32_0)) { 1588ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = pulldown_to_32(tree, 0, descr); 1589ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1590ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(descr & TREE_DESCR_32_0); 1591ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[2] = tree[0]; 1592ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~TREE_DESCR_32_0; 1593ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= (TREE_DESCR_16_1 | TREE_DESCR_16_0); 1594ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1595ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 4: case 6: 1596ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!(descr & TREE_DESCR_32_1)) { 1597ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = pulldown_to_32(tree, 4, descr); 1598ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1599ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(descr & TREE_DESCR_32_1); 1600ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[6] = tree[4]; 1601ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~TREE_DESCR_32_1; 1602ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= (TREE_DESCR_16_3 | TREE_DESCR_16_2); 1603ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1604ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown default: 1605ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 1606ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1607ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return descr; 1608ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1609ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1610ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UShort pulldown_to_8 ( /*MOD*/SVal* tree, UWord toff, UShort descr ) { 1611ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_16to8pulldown++; 1612ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown switch (toff) { 1613ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 0: case 1: 1614ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!(descr & TREE_DESCR_16_0)) { 1615ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = pulldown_to_16(tree, 0, descr); 1616ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1617ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(descr & TREE_DESCR_16_0); 1618ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[1] = tree[0]; 1619ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~TREE_DESCR_16_0; 1620ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= (TREE_DESCR_8_1 | TREE_DESCR_8_0); 1621ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1622ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 2: case 3: 1623ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!(descr & TREE_DESCR_16_1)) { 1624ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = pulldown_to_16(tree, 2, descr); 1625ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1626ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(descr & TREE_DESCR_16_1); 1627ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[3] = tree[2]; 1628ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~TREE_DESCR_16_1; 1629ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= (TREE_DESCR_8_3 | TREE_DESCR_8_2); 1630ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1631ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 4: case 5: 1632ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!(descr & TREE_DESCR_16_2)) { 1633ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = pulldown_to_16(tree, 4, descr); 1634ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1635ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(descr & TREE_DESCR_16_2); 1636ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[5] = tree[4]; 1637ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~TREE_DESCR_16_2; 1638ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= (TREE_DESCR_8_5 | TREE_DESCR_8_4); 1639ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1640ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 6: case 7: 1641ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!(descr & TREE_DESCR_16_3)) { 1642ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = pulldown_to_16(tree, 6, descr); 1643ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1644ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(descr & TREE_DESCR_16_3); 1645ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[7] = tree[6]; 1646ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~TREE_DESCR_16_3; 1647ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= (TREE_DESCR_8_7 | TREE_DESCR_8_6); 1648ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1649ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown default: 1650ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 1651ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1652ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return descr; 1653ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1654ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1655ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1656ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UShort pullup_descr_to_16 ( UShort descr, UWord toff ) { 1657ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort mask; 1658ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown switch (toff) { 1659ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 0: 1660ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mask = TREE_DESCR_8_1 | TREE_DESCR_8_0; 1661ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert( (descr & mask) == mask ); 1662ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~mask; 1663ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= TREE_DESCR_16_0; 1664ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1665ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 2: 1666ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mask = TREE_DESCR_8_3 | TREE_DESCR_8_2; 1667ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert( (descr & mask) == mask ); 1668ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~mask; 1669ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= TREE_DESCR_16_1; 1670ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1671ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 4: 1672ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mask = TREE_DESCR_8_5 | TREE_DESCR_8_4; 1673ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert( (descr & mask) == mask ); 1674ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~mask; 1675ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= TREE_DESCR_16_2; 1676ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1677ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 6: 1678ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mask = TREE_DESCR_8_7 | TREE_DESCR_8_6; 1679ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert( (descr & mask) == mask ); 1680ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~mask; 1681ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= TREE_DESCR_16_3; 1682ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1683ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown default: 1684ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 1685ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1686ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return descr; 1687ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1688ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1689ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UShort pullup_descr_to_32 ( UShort descr, UWord toff ) { 1690ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort mask; 1691ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown switch (toff) { 1692ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 0: 1693ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!(descr & TREE_DESCR_16_0)) 1694ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = pullup_descr_to_16(descr, 0); 1695ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!(descr & TREE_DESCR_16_1)) 1696ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = pullup_descr_to_16(descr, 2); 1697ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mask = TREE_DESCR_16_1 | TREE_DESCR_16_0; 1698ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert( (descr & mask) == mask ); 1699ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~mask; 1700ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= TREE_DESCR_32_0; 1701ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1702ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 4: 1703ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!(descr & TREE_DESCR_16_2)) 1704ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = pullup_descr_to_16(descr, 4); 1705ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!(descr & TREE_DESCR_16_3)) 1706ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = pullup_descr_to_16(descr, 6); 1707ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mask = TREE_DESCR_16_3 | TREE_DESCR_16_2; 1708ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert( (descr & mask) == mask ); 1709ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~mask; 1710ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= TREE_DESCR_32_1; 1711ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1712ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown default: 1713ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 1714ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1715ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return descr; 1716ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1717ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1718ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Bool valid_value_is_above_me_32 ( UShort descr, UWord toff ) { 1719ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown switch (toff) { 1720ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 0: case 4: 1721ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0 != (descr & TREE_DESCR_64); 1722ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown default: 1723ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 1724ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1725ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1726ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1727ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Bool valid_value_is_below_me_16 ( UShort descr, UWord toff ) { 1728ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown switch (toff) { 1729ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 0: 1730ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0 != (descr & (TREE_DESCR_8_1 | TREE_DESCR_8_0)); 1731ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 2: 1732ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0 != (descr & (TREE_DESCR_8_3 | TREE_DESCR_8_2)); 1733ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 4: 1734ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0 != (descr & (TREE_DESCR_8_5 | TREE_DESCR_8_4)); 1735ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 6: 1736ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0 != (descr & (TREE_DESCR_8_7 | TREE_DESCR_8_6)); 1737ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown default: 1738ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 1739ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1740ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1741ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1742ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* ------------ Cache management ------------ */ 1743ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1744ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_flush_cache ( void ) 1745ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1746ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown shmem__flush_and_invalidate_scache(); 1747ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1748ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1749ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1750ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_init ( void(*p_rcinc)(SVal), void(*p_rcdec)(SVal) ) 1751ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1752ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert( sizeof(UWord) == sizeof(Addr) ); 1753ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1754ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcinc = p_rcinc; 1755ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcdec = p_rcdec; 1756ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1757ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(map_shmem == NULL); 1758ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown map_shmem = VG_(newFM)( HG_(zalloc), "libhb.zsm_init.1 (map_shmem)", 1759ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free), 1760ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown NULL/*unboxed UWord cmp*/); 1761ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(map_shmem != NULL); 1762ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown shmem__invalidate_scache(); 1763ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1764ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* a SecMap must contain an integral number of CacheLines */ 1765ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0 == (N_SECMAP_ARANGE % N_LINE_ARANGE)); 1766ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* also ... a CacheLine holds an integral number of trees */ 1767ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0 == (N_LINE_ARANGE % 8)); 1768ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1769ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1770ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 1771ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 1772ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 1773ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// SECTION END compressed shadow memory // 1774ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 1775ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 1776ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 1777ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1778ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1779ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1780ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 1781ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 1782ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 1783ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// SECTION BEGIN vts primitives // 1784ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 1785ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 1786ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 1787ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1788ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1789b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* There's a 1-1 mapping between Thr and ThrIDs -- the latter merely 1790b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov being compact stand-ins for Thr*'s. Use these functions to map 1791b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov between them. */ 1792b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic ThrID Thr__to_ThrID ( Thr* thr ); /* fwds */ 1793b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic Thr* Thr__from_ThrID ( ThrID thrid ); /* fwds */ 1794b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1795b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov__attribute__((noreturn)) 1796b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void scalarts_limitations_fail_NORETURN ( Bool due_to_nThrs ) 1797b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 1798b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (due_to_nThrs) { 1799436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov const HChar* s = 1800b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "\n" 1801b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "Helgrind: cannot continue, run aborted: too many threads.\n" 1802b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "Sorry. Helgrind can only handle programs that create\n" 1803b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "%'llu or fewer threads over their entire lifetime.\n" 1804b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "\n"; 1805b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(umsg)(s, (ULong)(ThrID_MAX_VALID - 1024)); 1806b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } else { 1807436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov const HChar* s = 1808b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "\n" 1809b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "Helgrind: cannot continue, run aborted: too many\n" 1810b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "synchronisation events. Sorry. Helgrind can only handle\n" 1811b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "programs which perform %'llu or fewer\n" 1812b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "inter-thread synchronisation events (locks, unlocks, etc).\n" 1813b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "\n"; 1814b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(umsg)(s, (1ULL << SCALARTS_N_TYMBITS) - 1); 1815b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 1816b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(exit)(1); 1817b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /*NOTREACHED*/ 1818b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(0); /*wtf?!*/ 1819b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 1820b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1821b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1822b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* The dead thread (ThrID, actually) table. A thread may only be 1823b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov listed here if we have been notified thereof by libhb_async_exit. 1824b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov New entries are added at the end. The order isn't important, but 1825b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov the ThrID values must be unique. This table lists the identity of 1826b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov all threads that have ever died -- none are ever removed. We keep 1827b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov this table so as to be able to prune entries from VTSs. We don't 1828b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov actually need to keep the set of threads that have ever died -- 1829b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov only the threads that have died since the previous round of 1830b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov pruning. But it's useful for sanity check purposes to keep the 1831b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov entire set, so we do. */ 1832b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic XArray* /* of ThrID */ verydead_thread_table = NULL; 1833b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1834b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Arbitrary total ordering on ThrIDs. */ 1835436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovstatic Int cmp__ThrID ( const void* v1, const void* v2 ) { 1836436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov ThrID id1 = *(const ThrID*)v1; 1837436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov ThrID id2 = *(const ThrID*)v2; 1838b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (id1 < id2) return -1; 1839b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (id1 > id2) return 1; 1840b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return 0; 1841b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 1842b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1843b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void verydead_thread_table_init ( void ) 1844b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 1845b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(!verydead_thread_table); 1846b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov verydead_thread_table 1847b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov = VG_(newXA)( HG_(zalloc), 1848b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "libhb.verydead_thread_table_init.1", 1849b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov HG_(free), sizeof(ThrID) ); 1850b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(verydead_thread_table); 1851b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(setCmpFnXA)(verydead_thread_table, cmp__ThrID); 1852b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 1853b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1854ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1855ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* A VTS contains .ts, its vector clock, and also .id, a field to hold 1856ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a backlink for the caller's convenience. Since we have no idea 1857ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown what to set that to in the library, it always gets set to 1858ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID_INVALID. */ 1859ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef 1860ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct { 1861b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID id; 1862b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt usedTS; 1863b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt sizeTS; 1864b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS ts[0]; 1865ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1866ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VTS; 1867ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1868b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Allocate a VTS capable of storing 'sizeTS' entries. */ 1869436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovstatic VTS* VTS__new ( const HChar* who, UInt sizeTS ); 1870b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1871b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Make a clone of 'vts', sizing the new array to exactly match the 1872b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov number of ScalarTSs present. */ 1873436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovstatic VTS* VTS__clone ( const HChar* who, VTS* vts ); 1874ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1875b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Make a clone of 'vts' with the thrids in 'thrids' removed. The new 1876b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov array is sized exactly to hold the number of required elements. 1877b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 'thridsToDel' is an array of ThrIDs to be omitted in the clone, and 1878b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov must be in strictly increasing order. */ 1879436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovstatic VTS* VTS__subtract ( const HChar* who, VTS* vts, XArray* thridsToDel ); 1880ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1881ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Delete this VTS in its entirety. */ 1882ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void VTS__delete ( VTS* vts ); 1883ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1884b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Create a new singleton VTS in 'out'. Caller must have 1885b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov pre-allocated 'out' sufficiently big to hold the result in all 1886b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov possible cases. */ 1887b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void VTS__singleton ( /*OUT*/VTS* out, Thr* thr, ULong tym ); 1888ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1889b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Create in 'out' a VTS which is the same as 'vts' except with 1890b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov vts[me]++, so to speak. Caller must have pre-allocated 'out' 1891b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov sufficiently big to hold the result in all possible cases. */ 1892b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void VTS__tick ( /*OUT*/VTS* out, Thr* me, VTS* vts ); 1893ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1894b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Create in 'out' a VTS which is the join (max) of 'a' and 1895b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 'b'. Caller must have pre-allocated 'out' sufficiently big to hold 1896b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov the result in all possible cases. */ 1897b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void VTS__join ( /*OUT*/VTS* out, VTS* a, VTS* b ); 1898ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1899ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Compute the partial ordering relation of the two args. Although we 1900ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown could be completely general and return an enumeration value (EQ, 1901ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LT, GT, UN), in fact we only need LEQ, and so we may as well 1902ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown hardwire that fact. 1903ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1904b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Returns zero iff LEQ(A,B), or a valid ThrID if not (zero is an 1905b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov invald ThrID). In the latter case, the returned ThrID indicates 1906b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov the discovered point for which they are not. There may be more 1907b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov than one such point, but we only care about seeing one of them, not 1908b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov all of them. This rather strange convention is used because 1909b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov sometimes we want to know the actual index at which they first 1910b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov differ. */ 1911b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic UInt VTS__cmpLEQ ( VTS* a, VTS* b ); 1912ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1913ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Compute an arbitrary structural (total) ordering on the two args, 1914ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown based on their VCs, so they can be looked up in a table, tree, etc. 1915ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Returns -1, 0 or 1. */ 1916ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Word VTS__cmp_structural ( VTS* a, VTS* b ); 1917ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1918ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Debugging only. Display the given VTS in the buffer. */ 1919ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void VTS__show ( HChar* buf, Int nBuf, VTS* vts ); 1920ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1921ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Debugging only. Return vts[index], so to speak. */ 1922ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic ULong VTS__indexAt_SLOW ( VTS* vts, Thr* idx ); 1923ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1924b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Notify the VTS machinery that a thread has been declared 1925b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov comprehensively dead: that is, it has done an async exit AND it has 1926b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov been joined with. This should ensure that its local clocks (.viR 1927b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov and .viW) will never again change, and so all mentions of this 1928b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thread from all VTSs in the system may be removed. */ 1929b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void VTS__declare_thread_very_dead ( Thr* idx ); 1930ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1931ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--------------- to do with Vector Timestamps ---------------*/ 1932ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1933ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Bool is_sane_VTS ( VTS* vts ) 1934ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1935ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i, n; 1936ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ScalarTS *st1, *st2; 1937ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!vts) return False; 1938b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (vts->usedTS > vts->sizeTS) return False; 1939b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov n = vts->usedTS; 1940b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (n == 1) { 1941b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov st1 = &vts->ts[0]; 1942b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (st1->tym == 0) 1943b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return False; 1944b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 1945b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov else 1946ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n >= 2) { 1947ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < n-1; i++) { 1948b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov st1 = &vts->ts[i]; 1949b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov st2 = &vts->ts[i+1]; 1950b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (st1->thrid >= st2->thrid) 1951ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 1952ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (st1->tym == 0 || st2->tym == 0) 1953ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 1954ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1955ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1956ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return True; 1957ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1958ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1959ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1960ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Create a new, empty VTS. 1961ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 1962436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovstatic VTS* VTS__new ( const HChar* who, UInt sizeTS ) 19638f943afc22a6a683b78271836c8ddc462b4824a9Evgeniy Stepanov{ 1964b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS* vts = HG_(zalloc)(who, sizeof(VTS) + (sizeTS+1) * sizeof(ScalarTS)); 1965b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(vts->usedTS == 0); 1966b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov vts->sizeTS = sizeTS; 1967b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov *(ULong*)(&vts->ts[sizeTS]) = 0x0ddC0ffeeBadF00dULL; 19689bea4c13fca0e3bb4b719dcb3ed63d47d479294eKenny Root return vts; 19698f943afc22a6a683b78271836c8ddc462b4824a9Evgeniy Stepanov} 19708f943afc22a6a683b78271836c8ddc462b4824a9Evgeniy Stepanov 1971b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Clone this VTS. 1972b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov*/ 1973436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovstatic VTS* VTS__clone ( const HChar* who, VTS* vts ) 1974b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 1975b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(vts); 1976b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert( *(ULong*)(&vts->ts[vts->sizeTS]) == 0x0ddC0ffeeBadF00dULL); 1977b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt nTS = vts->usedTS; 1978b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS* clone = VTS__new(who, nTS); 1979b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov clone->id = vts->id; 1980b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov clone->sizeTS = nTS; 1981b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov clone->usedTS = nTS; 1982b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt i; 1983b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for (i = 0; i < nTS; i++) { 1984b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov clone->ts[i] = vts->ts[i]; 1985b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 1986b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert( *(ULong*)(&clone->ts[clone->sizeTS]) == 0x0ddC0ffeeBadF00dULL); 1987b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return clone; 1988b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 1989b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1990b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1991b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Make a clone of a VTS with specified ThrIDs removed. 'thridsToDel' 1992b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov must be in strictly increasing order. We could obviously do this 1993b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov much more efficiently (in linear time) if necessary. 1994b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov*/ 1995436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovstatic VTS* VTS__subtract ( const HChar* who, VTS* vts, XArray* thridsToDel ) 1996b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 1997b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt i, j; 1998b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(vts); 1999b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(thridsToDel); 2000b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert( *(ULong*)(&vts->ts[vts->sizeTS]) == 0x0ddC0ffeeBadF00dULL); 2001b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt nTS = vts->usedTS; 2002b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Figure out how many ScalarTSs will remain in the output. */ 2003b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt nReq = nTS; 2004b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for (i = 0; i < nTS; i++) { 2005b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID thrid = vts->ts[i].thrid; 2006b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (VG_(lookupXA)(thridsToDel, &thrid, NULL, NULL)) 2007b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov nReq--; 2008b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 2009b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(nReq <= nTS); 2010b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Copy the ones that will remain. */ 2011b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS* res = VTS__new(who, nReq); 2012b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov j = 0; 2013b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for (i = 0; i < nTS; i++) { 2014b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID thrid = vts->ts[i].thrid; 2015b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (VG_(lookupXA)(thridsToDel, &thrid, NULL, NULL)) 2016b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov continue; 2017b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov res->ts[j++] = vts->ts[i]; 2018b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 2019b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(j == nReq); 2020b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(j == res->sizeTS); 2021b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov res->usedTS = j; 2022b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert( *(ULong*)(&res->ts[j]) == 0x0ddC0ffeeBadF00dULL); 2023b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return res; 2024b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 2025b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2026ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2027ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Delete this VTS in its entirety. 2028ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 2029b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void VTS__delete ( VTS* vts ) 2030ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2031ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(vts); 2032b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(vts->usedTS <= vts->sizeTS); 2033b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert( *(ULong*)(&vts->ts[vts->sizeTS]) == 0x0ddC0ffeeBadF00dULL); 2034ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free)(vts); 2035ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2036ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2037ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2038ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Create a new singleton VTS. 2039ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 2040b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void VTS__singleton ( /*OUT*/VTS* out, Thr* thr, ULong tym ) 2041b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 2042ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(thr); 2043ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(tym >= 1); 2044b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(out); 2045b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(out->usedTS == 0); 2046b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(out->sizeTS >= 1); 2047b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt hi = out->usedTS++; 2048b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov out->ts[hi].thrid = Thr__to_ThrID(thr); 2049b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov out->ts[hi].tym = tym; 2050ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2051ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2052ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2053ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Return a new VTS in which vts[me]++, so to speak. 'vts' itself is 2054ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown not modified. 2055ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 2056b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void VTS__tick ( /*OUT*/VTS* out, Thr* me, VTS* vts ) 2057ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2058b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt i, n; 2059b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID me_thrid; 2060b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Bool found = False; 2061ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2062ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__vts__tick++; 2063ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2064b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(out); 2065b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(out->usedTS == 0); 2066b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (vts->usedTS >= ThrID_MAX_VALID) 2067b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov scalarts_limitations_fail_NORETURN( True/*due_to_nThrs*/ ); 2068b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(out->sizeTS >= 1 + vts->usedTS); 2069b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2070ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(me); 2071b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov me_thrid = Thr__to_ThrID(me); 2072ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_VTS(vts)); 2073b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov n = vts->usedTS; 2074ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2075b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Copy all entries which precede 'me'. */ 2076ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < n; i++) { 2077b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS* here = &vts->ts[i]; 2078b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (UNLIKELY(here->thrid >= me_thrid)) 2079ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 2080b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt hi = out->usedTS++; 2081b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov out->ts[hi] = *here; 2082b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 2083b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2084b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* 'i' now indicates the next entry to copy, if any. 2085b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov There are 3 possibilities: 2086b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov (a) there is no next entry (we used them all up already): 2087b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov add (me_thrid,1) to the output, and quit 2088b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov (b) there is a next entry, and its thrid > me_thrid: 2089b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov add (me_thrid,1) to the output, then copy the remaining entries 2090b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov (c) there is a next entry, and its thrid == me_thrid: 2091b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov copy it to the output but increment its timestamp value. 2092b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Then copy the remaining entries. (c) is the common case. 2093b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov */ 20949bea4c13fca0e3bb4b719dcb3ed63d47d479294eKenny Root tl_assert(i >= 0 && i <= n); 2095b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (i == n) { /* case (a) */ 2096b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt hi = out->usedTS++; 2097b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov out->ts[hi].thrid = me_thrid; 2098b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov out->ts[hi].tym = 1; 20999bea4c13fca0e3bb4b719dcb3ed63d47d479294eKenny Root } else { 2100b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* cases (b) and (c) */ 2101b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS* here = &vts->ts[i]; 2102b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (me_thrid == here->thrid) { /* case (c) */ 2103b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (UNLIKELY(here->tym >= (1ULL << SCALARTS_N_TYMBITS) - 2ULL)) { 2104b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* We're hosed. We have to stop. */ 2105b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov scalarts_limitations_fail_NORETURN( False/*!due_to_nThrs*/ ); 2106b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 2107b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt hi = out->usedTS++; 2108b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov out->ts[hi].thrid = here->thrid; 2109b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov out->ts[hi].tym = here->tym + 1; 2110b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov i++; 2111b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov found = True; 2112b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } else { /* case (b) */ 2113b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt hi = out->usedTS++; 2114b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov out->ts[hi].thrid = me_thrid; 2115b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov out->ts[hi].tym = 1; 2116b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 2117b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* And copy any remaining entries. */ 2118ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (/*keepgoing*/; i < n; i++) { 2119b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS* here2 = &vts->ts[i]; 2120b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt hi = out->usedTS++; 2121b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov out->ts[hi] = *here2; 2122ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2123ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2124b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2125b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(is_sane_VTS(out)); 2126b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(out->usedTS == vts->usedTS + (found ? 0 : 1)); 2127b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(out->usedTS <= out->sizeTS); 2128ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2129ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2130ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2131ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Return a new VTS constructed as the join (max) of the 2 args. 2132ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Neither arg is modified. 2133ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 2134b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void VTS__join ( /*OUT*/VTS* out, VTS* a, VTS* b ) 2135ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2136b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt ia, ib, useda, usedb; 2137ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ULong tyma, tymb, tymMax; 2138b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID thrid; 2139b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt ncommon = 0; 2140ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2141ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__vts__join++; 2142ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2143b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(a); 2144b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(b); 2145b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov useda = a->usedTS; 2146b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov usedb = b->usedTS; 2147b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2148b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(out); 2149b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(out->usedTS == 0); 2150b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* overly conservative test, but doing better involves comparing 2151b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov the two VTSs, which we don't want to do at this point. */ 2152b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (useda + usedb >= ThrID_MAX_VALID) 2153b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov scalarts_limitations_fail_NORETURN( True/*due_to_nThrs*/ ); 2154b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(out->sizeTS >= useda + usedb); 2155ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2156ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ia = ib = 0; 2157ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2158ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (1) { 2159ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2160b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* This logic is to enumerate triples (thrid, tyma, tymb) drawn 2161b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov from a and b in order, where thrid is the next ThrID 2162ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown occurring in either a or b, and tyma/b are the relevant 2163ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown scalar timestamps, taking into account implicit zeroes. */ 2164ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ia >= 0 && ia <= useda); 2165ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ib >= 0 && ib <= usedb); 2166ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2167ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (ia == useda && ib == usedb) { 2168ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* both empty - done */ 2169ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 2170ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2171ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else if (ia == useda && ib != usedb) { 2172ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* a empty, use up b */ 2173b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS* tmpb = &b->ts[ib]; 2174b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid = tmpb->thrid; 2175b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tyma = 0; 2176b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tymb = tmpb->tym; 2177ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ib++; 2178ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2179ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else if (ia != useda && ib == usedb) { 2180ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* b empty, use up a */ 2181b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS* tmpa = &a->ts[ia]; 2182b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid = tmpa->thrid; 2183b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tyma = tmpa->tym; 2184b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tymb = 0; 2185ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ia++; 2186ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2187ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 2188b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* both not empty; extract lowest-ThrID'd triple */ 2189b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS* tmpa = &a->ts[ia]; 2190b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS* tmpb = &b->ts[ib]; 2191b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (tmpa->thrid < tmpb->thrid) { 2192b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* a has the lowest unconsidered ThrID */ 2193b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid = tmpa->thrid; 2194b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tyma = tmpa->tym; 2195b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tymb = 0; 2196ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ia++; 2197b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } else if (tmpa->thrid > tmpb->thrid) { 2198b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* b has the lowest unconsidered ThrID */ 2199b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid = tmpb->thrid; 2200b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tyma = 0; 2201b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tymb = tmpb->tym; 2202ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ib++; 2203ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 2204b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* they both next mention the same ThrID */ 2205b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(tmpa->thrid == tmpb->thrid); 2206b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid = tmpa->thrid; /* == tmpb->thrid */ 2207b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tyma = tmpa->tym; 2208b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tymb = tmpb->tym; 2209ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ia++; 2210ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ib++; 2211b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ncommon++; 2212ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2213ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2214ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2215ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* having laboriously determined (thr, tyma, tymb), do something 2216ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown useful with it. */ 2217ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tymMax = tyma > tymb ? tyma : tymb; 2218ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tymMax > 0) { 2219b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt hi = out->usedTS++; 2220b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov out->ts[hi].thrid = thrid; 2221b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov out->ts[hi].tym = tymMax; 2222ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2223ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2224ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2225ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2226b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(is_sane_VTS(out)); 2227b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(out->usedTS <= out->sizeTS); 2228b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(out->usedTS == useda + usedb - ncommon); 2229ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2230ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2231ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2232b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Determine if 'a' <= 'b', in the partial ordering. Returns zero if 2233b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov they are, or the first ThrID for which they are not (no valid ThrID 2234b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov has the value zero). This rather strange convention is used 2235b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov because sometimes we want to know the actual index at which they 2236b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov first differ. */ 2237b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic UInt/*ThrID*/ VTS__cmpLEQ ( VTS* a, VTS* b ) 2238ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2239ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word ia, ib, useda, usedb; 2240ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ULong tyma, tymb; 2241ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2242ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__vts__cmpLEQ++; 2243ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2244b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(a); 2245b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(b); 2246b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov useda = a->usedTS; 2247b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov usedb = b->usedTS; 2248ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2249ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ia = ib = 0; 2250ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2251ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (1) { 2252ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2253ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* This logic is to enumerate doubles (tyma, tymb) drawn 2254ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown from a and b in order, and tyma/b are the relevant 2255ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown scalar timestamps, taking into account implicit zeroes. */ 2256b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID thrid; 2257ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2258ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ia >= 0 && ia <= useda); 2259ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ib >= 0 && ib <= usedb); 2260ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2261ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (ia == useda && ib == usedb) { 2262ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* both empty - done */ 2263ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 2264ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2265ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else if (ia == useda && ib != usedb) { 2266ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* a empty, use up b */ 2267b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS* tmpb = &b->ts[ib]; 2268b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tyma = 0; 2269b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tymb = tmpb->tym; 2270b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid = tmpb->thrid; 2271ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ib++; 2272ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2273ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else if (ia != useda && ib == usedb) { 2274ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* b empty, use up a */ 2275b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS* tmpa = &a->ts[ia]; 2276b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tyma = tmpa->tym; 2277b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid = tmpa->thrid; 2278b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tymb = 0; 2279ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ia++; 2280ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2281ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 2282b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* both not empty; extract lowest-ThrID'd triple */ 2283b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS* tmpa = &a->ts[ia]; 2284b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS* tmpb = &b->ts[ib]; 2285b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (tmpa->thrid < tmpb->thrid) { 2286b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* a has the lowest unconsidered ThrID */ 2287b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tyma = tmpa->tym; 2288b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid = tmpa->thrid; 2289b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tymb = 0; 2290ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ia++; 2291ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2292ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown else 2293b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (tmpa->thrid > tmpb->thrid) { 2294b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* b has the lowest unconsidered ThrID */ 2295b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tyma = 0; 2296b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tymb = tmpb->tym; 2297b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid = tmpb->thrid; 2298ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ib++; 2299ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 2300b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* they both next mention the same ThrID */ 2301b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(tmpa->thrid == tmpb->thrid); 2302b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tyma = tmpa->tym; 2303b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid = tmpa->thrid; 2304b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tymb = tmpb->tym; 2305ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ia++; 2306ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ib++; 2307ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2308ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2309ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2310ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* having laboriously determined (tyma, tymb), do something 2311ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown useful with it. */ 2312ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tyma > tymb) { 2313ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* not LEQ at this index. Quit, since the answer is 2314ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown determined already. */ 2315b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(thrid >= 1024); 2316b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return thrid; 2317ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2318ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2319ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2320b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return 0; /* all points are LEQ => return an invalid ThrID */ 2321ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2322ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2323ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2324ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Compute an arbitrary structural (total) ordering on the two args, 2325ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown based on their VCs, so they can be looked up in a table, tree, etc. 2326ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Returns -1, 0 or 1. (really just 'deriving Ord' :-) This can be 2327ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown performance critical so there is some effort expended to make it sa 2328ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fast as possible. 2329ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 2330ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownWord VTS__cmp_structural ( VTS* a, VTS* b ) 2331ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2332ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* We just need to generate an arbitrary total ordering based on 2333ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a->ts and b->ts. Preferably do it in a way which comes across likely 2334ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown differences relatively quickly. */ 2335ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i; 2336ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word useda = 0, usedb = 0; 2337ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ScalarTS *ctsa = NULL, *ctsb = NULL; 2338ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2339ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__vts__cmp_structural++; 2340ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2341ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(a); 2342ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(b); 2343ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2344b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ctsa = &a->ts[0]; useda = a->usedTS; 2345b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ctsb = &b->ts[0]; usedb = b->usedTS; 2346ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2347ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(useda == usedb)) { 2348ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ScalarTS *tmpa = NULL, *tmpb = NULL; 2349ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__vts__cmp_structural_slow++; 2350ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Same length vectors. Find the first difference, if any, as 2351ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fast as possible. */ 2352ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < useda; i++) { 2353ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tmpa = &ctsa[i]; 2354ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tmpb = &ctsb[i]; 2355b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (LIKELY(tmpa->tym == tmpb->tym 2356b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov && tmpa->thrid == tmpb->thrid)) 2357ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown continue; 2358ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown else 2359ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 2360ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2361ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(i == useda)) { 2362ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* They're identical. */ 2363ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0; 2364ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 2365ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(i >= 0 && i < useda); 2366ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tmpa->tym < tmpb->tym) return -1; 2367ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tmpa->tym > tmpb->tym) return 1; 2368b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (tmpa->thrid < tmpb->thrid) return -1; 2369b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (tmpa->thrid > tmpb->thrid) return 1; 2370ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* we just established them as non-identical, hence: */ 2371ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2372ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /*NOTREACHED*/ 2373ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 2374ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2375ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2376ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (useda < usedb) return -1; 2377ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (useda > usedb) return 1; 2378ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /*NOTREACHED*/ 2379ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 2380ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2381ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2382ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2383ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Debugging only. Display the given VTS in the buffer. 2384ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 2385b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovvoid VTS__show ( HChar* buf, Int nBuf, VTS* vts ) 2386b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 2387ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ScalarTS* st; 2388ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HChar unit[64]; 2389ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i, n; 2390ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Int avail = nBuf; 2391ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(vts && vts->ts); 2392ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(nBuf > 16); 2393ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown buf[0] = '['; 2394ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown buf[1] = 0; 2395b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov n = vts->usedTS; 2396ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < n; i++) { 2397ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(avail >= 40); 2398b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov st = &vts->ts[i]; 2399ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(memset)(unit, 0, sizeof(unit)); 2400b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(sprintf)(unit, i < n-1 ? "%u:%llu " : "%u:%llu", 2401b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov st->thrid, (ULong)st->tym); 2402ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (avail < VG_(strlen)(unit) + 40/*let's say*/) { 2403ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(strcat)(buf, " ...]"); 2404ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown buf[nBuf-1] = 0; 2405ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 2406ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2407ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(strcat)(buf, unit); 2408ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown avail -= VG_(strlen)(unit); 2409ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2410ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(strcat)(buf, "]"); 2411ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown buf[nBuf-1] = 0; 2412ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2413ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2414ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2415ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Debugging only. Return vts[index], so to speak. 2416ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 2417b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy StepanovULong VTS__indexAt_SLOW ( VTS* vts, Thr* idx ) 2418b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 2419ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i, n; 2420b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID idx_thrid = Thr__to_ThrID(idx); 2421ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__vts__indexat_slow++; 2422ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(vts && vts->ts); 2423b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov n = vts->usedTS; 2424ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < n; i++) { 2425b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS* st = &vts->ts[i]; 2426b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (st->thrid == idx_thrid) 2427ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return st->tym; 2428ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2429ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0; 2430ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2431ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2432ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2433b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* See comment on prototype above. 2434b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov*/ 2435b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void VTS__declare_thread_very_dead ( Thr* thr ) 2436b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 2437b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (0) VG_(printf)("VTQ: tae %p\n", thr); 2438b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2439b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(thr->llexit_done); 2440b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(thr->joinedwith_done); 2441b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2442b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID nyu; 2443b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov nyu = Thr__to_ThrID(thr); 2444b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(addToXA)( verydead_thread_table, &nyu ); 2445b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2446b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* We can only get here if we're assured that we'll never again 2447b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov need to look at this thread's ::viR or ::viW. Set them to 2448b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID_INVALID, partly so as to avoid holding on to the VTSs, but 2449b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov mostly so that we don't wind up pruning them (as that would be 2450b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov nonsensical: the only interesting ScalarTS entry for a dead 2451b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thread is its own index, and the pruning will remove that.). */ 2452b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID__rcdec(thr->viR); 2453b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID__rcdec(thr->viW); 2454b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thr->viR = VtsID_INVALID; 2455b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thr->viW = VtsID_INVALID; 2456b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 2457b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2458b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2459ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 2460ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 2461ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 2462ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// SECTION END vts primitives // 2463ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 2464ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 2465ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 2466ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2467ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2468ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2469ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 2470ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 2471ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 2472ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// SECTION BEGIN main library // 2473ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 2474ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 2475ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 2476ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2477ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2478ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 2479ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 2480ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// VTS set // 2481ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 2482ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 2483ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2484b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic WordFM* /* WordFM VTS* void */ vts_set = NULL; 2485ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2486ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void vts_set_init ( void ) 2487ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2488ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(!vts_set); 2489ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts_set = VG_(newFM)( HG_(zalloc), "libhb.vts_set_init.1", 2490ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free), 2491ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Word(*)(UWord,UWord))VTS__cmp_structural ); 2492ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(vts_set); 2493ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2494ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2495b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Given a VTS, look in vts_set to see if we already have a 2496b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov structurally identical one. If yes, return the pair (True, pointer 2497b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov to the existing one). If no, clone this one, add the clone to the 2498b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov set, and return (False, pointer to the clone). */ 2499b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic Bool vts_set__find__or__clone_and_add ( /*OUT*/VTS** res, VTS* cand ) 2500ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2501ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord keyW, valW; 2502b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov stats__vts_set__focaa++; 2503b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(cand->id == VtsID_INVALID); 2504ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* lookup cand (by value) */ 2505ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (VG_(lookupFM)( vts_set, &keyW, &valW, (UWord)cand )) { 2506ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* found it */ 2507ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(valW == 0); 2508ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* if this fails, cand (by ref) was already present (!) */ 2509ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(keyW != (UWord)cand); 2510b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov *res = (VTS*)keyW; 2511b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return True; 2512ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 2513b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* not present. Clone, add and return address of clone. */ 2514b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov stats__vts_set__focaa_a++; 2515b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS* clone = VTS__clone( "libhb.vts_set_focaa.1", cand ); 2516b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(clone != cand); 2517b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(addToFM)( vts_set, (UWord)clone, 0/*val is unused*/ ); 2518b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov *res = clone; 2519b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return False; 2520ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2521ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2522ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2523ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2524ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 2525ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 2526ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// VTS table // 2527ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 2528ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 2529ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2530ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void VtsID__invalidate_caches ( void ); /* fwds */ 2531ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2532ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* A type to hold VTS table entries. Invariants: 2533ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown If .vts == NULL, then this entry is not in use, so: 2534ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown - .rc == 0 2535ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown - this entry is on the freelist (unfortunately, does not imply 2536b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov any constraints on value for .freelink) 2537ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown If .vts != NULL, then this entry is in use: 2538ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown - .vts is findable in vts_set 2539ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown - .vts->id == this entry number 2540ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown - no specific value for .rc (even 0 is OK) 2541b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov - this entry is not on freelist, so .freelink == VtsID_INVALID 2542ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 2543ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef 2544ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct { 2545ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VTS* vts; /* vts, in vts_set */ 2546ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord rc; /* reference count - enough for entire aspace */ 2547ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID freelink; /* chain for free entries, VtsID_INVALID at end */ 2548b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID remap; /* used only during pruning */ 2549ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2550ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsTE; 2551ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2552ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* The VTS table. */ 2553ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic XArray* /* of VtsTE */ vts_tab = NULL; 2554ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2555ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* An index into the VTS table, indicating the start of the list of 2556ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown free (available for use) entries. If the list is empty, this is 2557ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID_INVALID. */ 2558ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic VtsID vts_tab_freelist = VtsID_INVALID; 2559ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2560ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Do a GC of vts_tab when the freelist becomes empty AND the size of 2561ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts_tab equals or exceeds this size. After GC, the value here is 2562ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown set appropriately so as to check for the next GC point. */ 2563ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Word vts_next_GC_at = 1000; 2564ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2565ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void vts_tab_init ( void ) 2566ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2567ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts_tab 2568ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown = VG_(newXA)( HG_(zalloc), "libhb.vts_tab_init.1", 2569ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free), sizeof(VtsTE) ); 2570ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts_tab_freelist 2571ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown = VtsID_INVALID; 2572ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(vts_tab); 2573ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2574ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2575ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Add ii to the free list, checking that it looks out-of-use. */ 2576ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void add_to_free_list ( VtsID ii ) 2577ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2578ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsTE* ie = VG_(indexXA)( vts_tab, ii ); 2579ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ie->vts == NULL); 2580ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ie->rc == 0); 2581ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ie->freelink == VtsID_INVALID); 2582ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ie->freelink = vts_tab_freelist; 2583ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts_tab_freelist = ii; 2584ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2585ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2586ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Get an entry from the free list. This will return VtsID_INVALID if 2587ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown the free list is empty. */ 2588ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic VtsID get_from_free_list ( void ) 2589ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2590ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID ii; 2591ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsTE* ie; 2592ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (vts_tab_freelist == VtsID_INVALID) 2593ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return VtsID_INVALID; 2594ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ii = vts_tab_freelist; 2595ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ie = VG_(indexXA)( vts_tab, ii ); 2596ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ie->vts == NULL); 2597ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ie->rc == 0); 2598ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts_tab_freelist = ie->freelink; 2599ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ii; 2600ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2601ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2602ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Produce a new VtsID that can be used, either by getting it from 2603ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown the freelist, or, if that is empty, by expanding vts_tab. */ 2604ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic VtsID get_new_VtsID ( void ) 2605ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2606ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID ii; 2607ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsTE te; 2608ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ii = get_from_free_list(); 2609ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (ii != VtsID_INVALID) 2610ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ii; 2611ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown te.vts = NULL; 2612ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown te.rc = 0; 2613ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown te.freelink = VtsID_INVALID; 2614b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov te.remap = VtsID_INVALID; 2615ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ii = (VtsID)VG_(addToXA)( vts_tab, &te ); 2616ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ii; 2617ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2618ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2619ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2620ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Indirect callback from lib_zsm. */ 2621ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void VtsID__rcinc ( VtsID ii ) 2622ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2623ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsTE* ie; 2624ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* VG_(indexXA) does a range check for us */ 2625ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ie = VG_(indexXA)( vts_tab, ii ); 2626ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ie->vts); /* else it's not in use */ 2627ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ie->rc < ~0UL); /* else we can't continue */ 2628ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ie->vts->id == ii); 2629ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ie->rc++; 2630ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2631ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2632ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Indirect callback from lib_zsm. */ 2633ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void VtsID__rcdec ( VtsID ii ) 2634ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2635ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsTE* ie; 2636ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* VG_(indexXA) does a range check for us */ 2637ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ie = VG_(indexXA)( vts_tab, ii ); 2638ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ie->vts); /* else it's not in use */ 2639ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ie->rc > 0); /* else RC snafu */ 2640ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ie->vts->id == ii); 2641ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ie->rc--; 2642ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2643ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2644ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2645b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Look up 'cand' in our collection of VTSs. If present, return the 2646b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID for the pre-existing version. If not present, clone it, add 2647b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov the clone to both vts_tab and vts_set, allocate a fresh VtsID for 2648b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov it, and return that. */ 2649b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic VtsID vts_tab__find__or__clone_and_add ( VTS* cand ) 2650ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2651b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS* in_tab = NULL; 2652ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(cand->id == VtsID_INVALID); 2653b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Bool already_have = vts_set__find__or__clone_and_add( &in_tab, cand ); 2654b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(in_tab); 2655b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (already_have) { 2656b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* We already have a copy of 'cand'. Use that. */ 2657ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsTE* ie; 2658b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(in_tab->id != VtsID_INVALID); 2659b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ie = VG_(indexXA)( vts_tab, in_tab->id ); 2660b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(ie->vts == in_tab); 2661b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return in_tab->id; 2662ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 2663ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID ii = get_new_VtsID(); 2664ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsTE* ie = VG_(indexXA)( vts_tab, ii ); 2665b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ie->vts = in_tab; 2666ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ie->rc = 0; 2667ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ie->freelink = VtsID_INVALID; 2668b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov in_tab->id = ii; 2669ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ii; 2670ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2671ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2672ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2673ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2674436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovstatic void show_vts_stats ( const HChar* caller ) 2675ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2676ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord nSet, nTab, nLive; 2677ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ULong totrc; 2678ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord n, i; 2679ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nSet = VG_(sizeFM)( vts_set ); 2680ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nTab = VG_(sizeXA)( vts_tab ); 2681ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown totrc = 0; 2682ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nLive = 0; 2683ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown n = VG_(sizeXA)( vts_tab ); 2684ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < n; i++) { 2685ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsTE* ie = VG_(indexXA)( vts_tab, i ); 2686ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (ie->vts) { 2687ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nLive++; 2688ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown totrc += (ULong)ie->rc; 2689ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 2690ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ie->rc == 0); 2691ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2692ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2693ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" show_vts_stats %s\n", caller); 2694ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" vts_tab size %4lu\n", nTab); 2695ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" vts_tab live %4lu\n", nLive); 2696ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" vts_set size %4lu\n", nSet); 2697ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" total rc %4llu\n", totrc); 2698ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2699ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2700b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2701b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* --- Helpers for VtsID pruning --- */ 2702b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2703b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic 2704b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovvoid remap_VtsID ( /*MOD*/XArray* /* of VtsTE */ old_tab, 2705b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /*MOD*/XArray* /* of VtsTE */ new_tab, 2706b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID* ii ) 2707b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 2708b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsTE *old_te, *new_te; 2709b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID old_id, new_id; 2710b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* We're relying here on VG_(indexXA)'s range checking to assert on 2711b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov any stupid values, in particular *ii == VtsID_INVALID. */ 2712b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov old_id = *ii; 2713b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov old_te = VG_(indexXA)( old_tab, old_id ); 2714b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov old_te->rc--; 2715b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov new_id = old_te->remap; 2716b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov new_te = VG_(indexXA)( new_tab, new_id ); 2717b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov new_te->rc++; 2718b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov *ii = new_id; 2719b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 2720b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2721b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic 2722b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovvoid remap_VtsIDs_in_SVal ( /*MOD*/XArray* /* of VtsTE */ old_tab, 2723b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /*MOD*/XArray* /* of VtsTE */ new_tab, 2724b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov SVal* s ) 2725b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 2726b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov SVal old_sv, new_sv; 2727b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov old_sv = *s; 2728b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (SVal__isC(old_sv)) { 2729b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID rMin, wMin; 2730b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov rMin = SVal__unC_Rmin(old_sv); 2731b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov wMin = SVal__unC_Wmin(old_sv); 2732b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov remap_VtsID( old_tab, new_tab, &rMin ); 2733b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov remap_VtsID( old_tab, new_tab, &wMin ); 2734b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov new_sv = SVal__mkC( rMin, wMin ); 2735b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov *s = new_sv; 2736b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 2737b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 2738b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2739b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2740ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* NOT TO BE CALLED FROM WITHIN libzsm. */ 2741ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown__attribute__((noinline)) 2742ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void vts_tab__do_GC ( Bool show_stats ) 2743ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2744ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i, nTab, nLive, nFreed; 2745ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2746b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* ---------- BEGIN VTS GC ---------- */ 2747ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* check this is actually necessary. */ 2748ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(vts_tab_freelist == VtsID_INVALID); 2749ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2750ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* empty the caches for partial order checks and binary joins. We 2751ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown could do better and prune out the entries to be deleted, but it 2752ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ain't worth the hassle. */ 2753ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__invalidate_caches(); 2754ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2755ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* First, make the reference counts up to date. */ 2756ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_flush_cache(); 2757ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2758ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nTab = VG_(sizeXA)( vts_tab ); 2759ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2760ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (show_stats) { 2761ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("<<GC begins at vts_tab size %lu>>\n", nTab); 2762ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown show_vts_stats("before GC"); 2763ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2764ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2765b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Now we can inspect the entire vts_tab. Any entries with zero 2766b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov .rc fields are now no longer in use and can be put back on the 2767ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown free list, removed from vts_set, and deleted. */ 2768ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nFreed = 0; 2769ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < nTab; i++) { 2770ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool present; 2771b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UWord oldK = 0, oldV = 12345; 2772ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsTE* te = VG_(indexXA)( vts_tab, i ); 2773ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (te->vts == NULL) { 2774ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(te->rc == 0); 2775ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown continue; /* already on the free list (presumably) */ 2776ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2777ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (te->rc > 0) 2778ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown continue; /* in use */ 2779ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Ok, we got one we can free. */ 2780ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(te->vts->id == i); 2781ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* first, remove it from vts_set. */ 2782ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown present = VG_(delFromFM)( vts_set, 2783ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown &oldK, &oldV, (UWord)te->vts ); 2784ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(present); /* else it isn't in vts_set ?! */ 2785ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(oldV == 0); /* no info stored in vts_set val fields */ 2786ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(oldK == (UWord)te->vts); /* else what did delFromFM find?! */ 2787ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* now free the VTS itself */ 2788ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VTS__delete(te->vts); 2789ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown te->vts = NULL; 2790ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* and finally put this entry on the free list */ 2791ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(te->freelink == VtsID_INVALID); /* can't already be on it */ 2792ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown add_to_free_list( i ); 2793ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nFreed++; 2794ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2795ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2796ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Now figure out when the next GC should be. We'll allow the 2797ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown number of VTSs to double before GCing again. Except of course 2798ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown that since we can't (or, at least, don't) shrink vts_tab, we 2799ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown can't set the threshhold value smaller than it. */ 2800ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(nFreed <= nTab); 2801ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nLive = nTab - nFreed; 2802ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(nLive >= 0 && nLive <= nTab); 2803ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts_next_GC_at = 2 * nLive; 2804ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (vts_next_GC_at < nTab) 2805ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts_next_GC_at = nTab; 2806ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2807ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (show_stats) { 2808ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown show_vts_stats("after GC"); 2809ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("<<GC ends, next gc at %ld>>\n", vts_next_GC_at); 2810ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2811ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2812ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (VG_(clo_stats)) { 2813b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov static UInt ctr = 1; 2814ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(nTab > 0); 2815ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(message)(Vg_DebugMsg, 2816ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "libhb: VTS GC: #%u old size %lu live %lu (%2llu%%)\n", 2817ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ctr++, nTab, nLive, (100ULL * (ULong)nLive) / (ULong)nTab); 2818ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2819b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* ---------- END VTS GC ---------- */ 2820b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2821b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Decide whether to do VTS pruning. We have one of three 2822b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov settings. */ 2823b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov static UInt pruning_auto_ctr = 0; /* do not make non-static */ 2824b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2825b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Bool do_pruning = False; 2826b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov switch (HG_(clo_vts_pruning)) { 2827b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov case 0: /* never */ 2828b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov break; 2829b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov case 1: /* auto */ 2830b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov do_pruning = (++pruning_auto_ctr % 5) == 0; 2831b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov break; 2832b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov case 2: /* always */ 2833b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov do_pruning = True; 2834b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov break; 2835b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov default: 2836b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(0); 2837b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 2838b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2839b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* The rest of this routine only handles pruning, so we can 2840b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov quit at this point if it is not to be done. */ 2841b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (!do_pruning) 2842b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return; 2843b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2844b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* ---------- BEGIN VTS PRUNING ---------- */ 2845b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* We begin by sorting the backing table on its .thr values, so as 2846b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov to (1) check they are unique [else something has gone wrong, 2847b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov since it means we must have seen some Thr* exiting more than 2848b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov once, which can't happen], and (2) so that we can quickly look 2849b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov up the dead-thread entries as we work through the VTSs. */ 2850b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(sortXA)( verydead_thread_table ); 2851b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Sanity check: check for unique .sts.thr values. */ 2852b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UWord nBT = VG_(sizeXA)( verydead_thread_table ); 2853b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (nBT > 0) { 2854b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID thrid1, thrid2; 2855b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid2 = *(ThrID*)VG_(indexXA)( verydead_thread_table, 0 ); 2856b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for (i = 1; i < nBT; i++) { 2857b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid1 = thrid2; 2858b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid2 = *(ThrID*)VG_(indexXA)( verydead_thread_table, i ); 2859b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(thrid1 < thrid2); 2860b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 2861b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 2862b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Ok, so the dead thread table has unique and in-order keys. */ 2863b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2864b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* We will run through the old table, and create a new table and 2865b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov set, at the same time setting the .remap entries in the old 2866b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov table to point to the new entries. Then, visit every VtsID in 2867b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov the system, and replace all of them with new ones, using the 2868b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov .remap entries in the old table. Finally, we can delete the old 2869b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov table and set. */ 2870b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2871b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov XArray* /* of VtsTE */ new_tab 2872b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov = VG_(newXA)( HG_(zalloc), "libhb.vts_tab__do_GC.new_tab", 2873b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov HG_(free), sizeof(VtsTE) ); 2874b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2875b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* WordFM VTS* void */ 2876b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov WordFM* new_set 2877b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov = VG_(newFM)( HG_(zalloc), "libhb.vts_tab__do_GC.new_set", 2878b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov HG_(free), 2879b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov (Word(*)(UWord,UWord))VTS__cmp_structural ); 2880b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2881b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Visit each old VTS. For each one: 2882b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2883b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov * make a pruned version 2884b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2885b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov * search new_set for the pruned version, yielding either 2886b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Nothing (not present) or the new VtsID for it. 2887b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2888b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov * if not present, allocate a new VtsID for it, insert (pruned 2889b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS, new VtsID) in the tree, and set 2890b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov remap_table[old VtsID] = new VtsID. 2891b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2892b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov * if present, set remap_table[old VtsID] = new VtsID, where 2893b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov new VtsID was determined by the tree lookup. Then free up 2894b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov the clone. 2895b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov */ 2896b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2897b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UWord nBeforePruning = 0, nAfterPruning = 0; 2898b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UWord nSTSsBefore = 0, nSTSsAfter = 0; 2899b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID new_VtsID_ctr = 0; 2900b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2901b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for (i = 0; i < nTab; i++) { 2902b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2903b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* For each old VTS .. */ 2904b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsTE* old_te = VG_(indexXA)( vts_tab, i ); 2905b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS* old_vts = old_te->vts; 2906b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(old_te->remap == VtsID_INVALID); 2907b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2908b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Skip it if not in use */ 2909b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (old_te->rc == 0) { 2910b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(old_vts == NULL); 2911b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov continue; 2912b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 2913b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(old_vts != NULL); 2914b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(old_vts->id == i); 2915b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(old_vts->ts != NULL); 2916b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2917b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* It is in use. Make a pruned version. */ 2918b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov nBeforePruning++; 2919b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov nSTSsBefore += old_vts->usedTS; 2920b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS* new_vts = VTS__subtract("libhb.vts_tab__do_GC.new_vts", 2921b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov old_vts, verydead_thread_table); 2922b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(new_vts->sizeTS == new_vts->usedTS); 2923b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(*(ULong*)(&new_vts->ts[new_vts->usedTS]) 2924b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov == 0x0ddC0ffeeBadF00dULL); 2925b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2926b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Get rid of the old VTS and the tree entry. It's a bit more 2927b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov complex to incrementally delete the VTSs now than to nuke 2928b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov them all after we're done, but the upside is that we don't 2929b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov wind up temporarily storing potentially two complete copies 2930b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov of each VTS and hence spiking memory use. */ 2931b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UWord oldK = 0, oldV = 12345; 2932b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Bool present = VG_(delFromFM)( vts_set, 2933b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov &oldK, &oldV, (UWord)old_vts ); 2934b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(present); /* else it isn't in vts_set ?! */ 2935b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(oldV == 0); /* no info stored in vts_set val fields */ 2936b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(oldK == (UWord)old_vts); /* else what did delFromFM find?! */ 2937b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* now free the VTS itself */ 2938b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS__delete(old_vts); 2939b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov old_te->vts = NULL; 2940b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov old_vts = NULL; 2941b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2942b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* NO MENTIONS of old_vts allowed beyond this point. */ 2943b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2944b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Ok, we have the pruned copy in new_vts. See if a 2945b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov structurally identical version is already present in new_set. 2946b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov If so, delete the one we just made and move on; if not, add 2947b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov it. */ 2948b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS* identical_version = NULL; 2949b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UWord valW = 12345; 2950b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (VG_(lookupFM)(new_set, (UWord*)&identical_version, &valW, 2951b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov (UWord)new_vts)) { 2952b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // already have it 2953b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(valW == 0); 2954b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(identical_version != NULL); 2955b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(identical_version != new_vts); 2956b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS__delete(new_vts); 2957b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov new_vts = identical_version; 2958b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(new_vts->id != VtsID_INVALID); 2959b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } else { 2960b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(valW == 12345); 2961b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(identical_version == NULL); 2962b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov new_vts->id = new_VtsID_ctr++; 2963b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Bool b = VG_(addToFM)(new_set, (UWord)new_vts, 0); 2964b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(!b); 2965b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsTE new_te; 2966b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov new_te.vts = new_vts; 2967b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov new_te.rc = 0; 2968b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov new_te.freelink = VtsID_INVALID; 2969b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov new_te.remap = VtsID_INVALID; 2970b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Word j = VG_(addToXA)( new_tab, &new_te ); 2971b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(j <= i); 2972b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(j == new_VtsID_ctr - 1); 2973b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // stats 2974b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov nAfterPruning++; 2975b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov nSTSsAfter += new_vts->usedTS; 2976b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 2977b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov old_te->remap = new_vts->id; 2978b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2979b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } /* for (i = 0; i < nTab; i++) */ 2980b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2981b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* At this point, we have: 2982b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov * the old VTS table, with its .remap entries set, 2983b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov and with all .vts == NULL. 2984b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov * the old VTS tree should be empty, since it and the old VTSs 2985b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov it contained have been incrementally deleted was we worked 2986b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov through the old table. 2987b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov * the new VTS table, with all .rc == 0, all .freelink and .remap 2988b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov == VtsID_INVALID. 2989b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov * the new VTS tree. 2990b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov */ 2991b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert( VG_(sizeFM)(vts_set) == 0 ); 2992b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2993b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Now actually apply the mapping. */ 2994b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Visit all the VtsIDs in the entire system. Where do we expect 2995b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov to find them? 2996b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov (a) in shadow memory -- the LineZs and LineFs 2997b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov (b) in our collection of struct _Thrs. 2998b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov (c) in our collection of struct _SOs. 2999b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Nowhere else, AFAICS. Not in the zsm cache, because that just 3000b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov got invalidated. 3001b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3002b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Using the .remap fields in vts_tab, map each old VtsID to a new 3003b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID. For each old VtsID, dec its rc; and for each new one, 3004b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov inc it. This sets up the new refcounts, and it also gives a 3005b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cheap sanity check of the old ones: all old refcounts should be 3006b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov zero after this operation. 3007b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov */ 3008b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3009b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Do the mappings for (a) above: iterate over the Primary shadow 3010b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov mem map (WordFM Addr SecMap*). */ 3011b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UWord secmapW = 0; 3012b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(initIterFM)( map_shmem ); 3013b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov while (VG_(nextIterFM)( map_shmem, NULL, &secmapW )) { 3014b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UWord j; 3015b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov SecMap* sm = (SecMap*)secmapW; 3016b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(sm->magic == SecMap_MAGIC); 3017b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Deal with the LineZs */ 3018b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for (i = 0; i < N_SECMAP_ZLINES; i++) { 3019b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov LineZ* lineZ = &sm->linesZ[i]; 3020b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (lineZ->dict[0] == SVal_INVALID) 3021b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov continue; /* not in use -- data is in F rep instead */ 3022b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for (j = 0; j < 4; j++) 3023b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov remap_VtsIDs_in_SVal(vts_tab, new_tab, &lineZ->dict[j]); 3024b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 3025b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Deal with the LineFs */ 3026b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for (i = 0; i < sm->linesF_size; i++) { 3027b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov LineF* lineF = &sm->linesF[i]; 3028b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (!lineF->inUse) 3029b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov continue; 3030b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for (j = 0; j < N_LINE_ARANGE; j++) 3031b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov remap_VtsIDs_in_SVal(vts_tab, new_tab, &lineF->w64s[j]); 3032b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 3033b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 3034b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(doneIterFM)( map_shmem ); 3035b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3036b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Do the mappings for (b) above: visit our collection of struct 3037b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov _Thrs. */ 3038b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Thread* hgthread = get_admin_threads(); 3039b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(hgthread); 3040b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov while (hgthread) { 3041b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Thr* hbthr = hgthread->hbthr; 3042b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(hbthr); 3043b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Threads that are listed in the prunable set have their viR 3044b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov and viW set to VtsID_INVALID, so we can't mess with them. */ 3045b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (hbthr->llexit_done && hbthr->joinedwith_done) { 3046b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(hbthr->viR == VtsID_INVALID); 3047b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(hbthr->viW == VtsID_INVALID); 3048b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov hgthread = hgthread->admin; 3049b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov continue; 3050b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 3051b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov remap_VtsID( vts_tab, new_tab, &hbthr->viR ); 3052b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov remap_VtsID( vts_tab, new_tab, &hbthr->viW ); 3053b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov hgthread = hgthread->admin; 3054b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 3055b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3056b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Do the mappings for (c) above: visit the struct _SOs. */ 3057b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov SO* so = admin_SO; 3058b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov while (so) { 3059b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (so->viR != VtsID_INVALID) 3060b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov remap_VtsID( vts_tab, new_tab, &so->viR ); 3061b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (so->viW != VtsID_INVALID) 3062b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov remap_VtsID( vts_tab, new_tab, &so->viW ); 3063b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov so = so->admin_next; 3064b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 3065b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3066b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* So, we're nearly done (with this incredibly complex operation). 3067b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Check the refcounts for the old VtsIDs all fell to zero, as 3068b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov expected. Any failure is serious. */ 3069b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for (i = 0; i < nTab; i++) { 3070b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsTE* te = VG_(indexXA)( vts_tab, i ); 3071b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(te->vts == NULL); 3072b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* This is the assert proper. Note we're also asserting 3073b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov zeroness for old entries which are unmapped (hence have 3074b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov .remap == VtsID_INVALID). That's OK. */ 3075b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(te->rc == 0); 3076b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 3077b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3078b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Install the new table and set. */ 3079b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(deleteFM)(vts_set, NULL/*kFin*/, NULL/*vFin*/); 3080b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov vts_set = new_set; 3081b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(deleteXA)( vts_tab ); 3082b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov vts_tab = new_tab; 3083b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3084b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* The freelist of vts_tab entries is empty now, because we've 3085b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov compacted all of the live entries at the low end of the 3086b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov table. */ 3087b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov vts_tab_freelist = VtsID_INVALID; 3088b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3089b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Sanity check vts_set and vts_tab. */ 3090b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3091b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Because all the live entries got slid down to the bottom of vts_tab: */ 3092b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert( VG_(sizeXA)( vts_tab ) == VG_(sizeFM)( vts_set )); 3093b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3094b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Assert that the vts_tab and vts_set entries point at each other 3095b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov in the required way */ 3096b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UWord wordK = 0, wordV = 0; 3097b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(initIterFM)( vts_set ); 3098b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov while (VG_(nextIterFM)( vts_set, &wordK, &wordV )) { 3099b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(wordK != 0); 3100b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(wordV == 0); 3101b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS* vts = (VTS*)wordK; 3102b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(vts->id != VtsID_INVALID); 3103b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsTE* te = VG_(indexXA)( vts_tab, vts->id ); 3104b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(te->vts == vts); 3105b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 3106b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(doneIterFM)( vts_set ); 3107b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3108b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Also iterate over the table, and check each entry is 3109b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov plausible. */ 3110b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov nTab = VG_(sizeXA)( vts_tab ); 3111b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for (i = 0; i < nTab; i++) { 3112b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsTE* te = VG_(indexXA)( vts_tab, i ); 3113b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(te->vts); 3114b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(te->vts->id == i); 3115b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(te->rc > 0); /* 'cos we just GC'd */ 3116b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(te->freelink == VtsID_INVALID); /* in use */ 3117b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(te->remap == VtsID_INVALID); /* not relevant */ 3118b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 3119b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3120b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* And we're done. Bwahahaha. Ha. Ha. Ha. */ 3121b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (VG_(clo_stats)) { 3122b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov static UInt ctr = 1; 3123b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(nTab > 0); 3124b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(message)( 3125b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Vg_DebugMsg, 3126b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "libhb: VTS PR: #%u before %lu (avg sz %lu) " 3127b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "after %lu (avg sz %lu)\n", 3128b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ctr++, 3129b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov nBeforePruning, nSTSsBefore / (nBeforePruning ? nBeforePruning : 1), 3130b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov nAfterPruning, nSTSsAfter / (nAfterPruning ? nAfterPruning : 1) 3131b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ); 3132b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 3133b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (0) 3134b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(printf)("VTQ: before pruning %lu (avg sz %lu), " 3135b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "after pruning %lu (avg sz %lu)\n", 3136b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov nBeforePruning, nSTSsBefore / nBeforePruning, 3137b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov nAfterPruning, nSTSsAfter / nAfterPruning); 3138b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* ---------- END VTS PRUNING ---------- */ 3139ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3140ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3141ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3142ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 3143ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 3144ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Vts IDs // 3145ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 3146ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 3147ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3148ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown////////////////////////// 3149b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* A temporary, max-sized VTS which is used as a temporary (the first 3150b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov argument) in VTS__singleton, VTS__tick and VTS__join operations. */ 3151b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic VTS* temp_max_sized_VTS = NULL; 3152b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3153b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov////////////////////////// 3154ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic ULong stats__cmpLEQ_queries = 0; 3155ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic ULong stats__cmpLEQ_misses = 0; 3156ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic ULong stats__join2_queries = 0; 3157ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic ULong stats__join2_misses = 0; 3158ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3159ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline UInt ROL32 ( UInt w, Int n ) { 3160ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown w = (w << n) | (w >> (32-n)); 3161ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return w; 3162ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3163ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline UInt hash_VtsIDs ( VtsID vi1, VtsID vi2, UInt nTab ) { 3164ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt hash = ROL32(vi1,19) ^ ROL32(vi2,13); 3165ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return hash % nTab; 3166ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3167ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3168ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_CMPLEQ_CACHE 1023 3169ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 3170ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct { VtsID vi1; VtsID vi2; Bool leq; } 3171ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cmpLEQ_cache[N_CMPLEQ_CACHE]; 3172ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3173ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_JOIN2_CACHE 1023 3174ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 3175ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct { VtsID vi1; VtsID vi2; VtsID res; } 3176ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown join2_cache[N_JOIN2_CACHE]; 3177ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3178ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void VtsID__invalidate_caches ( void ) { 3179ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Int i; 3180ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_CMPLEQ_CACHE; i++) { 3181ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cmpLEQ_cache[i].vi1 = VtsID_INVALID; 3182ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cmpLEQ_cache[i].vi2 = VtsID_INVALID; 3183ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cmpLEQ_cache[i].leq = False; 3184ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3185ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_JOIN2_CACHE; i++) { 3186ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown join2_cache[i].vi1 = VtsID_INVALID; 3187ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown join2_cache[i].vi2 = VtsID_INVALID; 3188ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown join2_cache[i].res = VtsID_INVALID; 3189ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3190ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3191ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown////////////////////////// 3192ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3193ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//static Bool VtsID__is_valid ( VtsID vi ) { 3194ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// VtsTE* ve; 3195ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// if (vi >= (VtsID)VG_(sizeXA)( vts_tab )) 3196ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// return False; 3197ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// ve = VG_(indexXA)( vts_tab, vi ); 3198ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// if (!ve->vts) 3199ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// return False; 3200ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// tl_assert(ve->vts->id == vi); 3201ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// return True; 3202ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//} 3203ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3204ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic VTS* VtsID__to_VTS ( VtsID vi ) { 3205ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsTE* te = VG_(indexXA)( vts_tab, vi ); 3206ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(te->vts); 3207ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return te->vts; 3208ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3209ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3210ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void VtsID__pp ( VtsID vi ) { 3211ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HChar buf[100]; 3212ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VTS* vts = VtsID__to_VTS(vi); 3213ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VTS__show( buf, sizeof(buf)-1, vts ); 3214ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown buf[sizeof(buf)-1] = 0; 3215ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s", buf); 3216ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3217ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3218ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* compute partial ordering relation of vi1 and vi2. */ 3219ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown__attribute__((noinline)) 3220ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Bool VtsID__cmpLEQ_WRK ( VtsID vi1, VtsID vi2 ) { 3221ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt hash; 3222ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool leq; 3223ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VTS *v1, *v2; 3224ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //if (vi1 == vi2) return True; 3225ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(vi1 != vi2); 3226ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ////++ 3227ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cmpLEQ_queries++; 3228ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown hash = hash_VtsIDs(vi1, vi2, N_CMPLEQ_CACHE); 3229ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (cmpLEQ_cache[hash].vi1 == vi1 3230ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown && cmpLEQ_cache[hash].vi2 == vi2) 3231ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return cmpLEQ_cache[hash].leq; 3232ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cmpLEQ_misses++; 3233ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ////-- 3234ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown v1 = VtsID__to_VTS(vi1); 3235ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown v2 = VtsID__to_VTS(vi2); 3236b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov leq = VTS__cmpLEQ( v1, v2 ) == 0; 3237ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ////++ 3238ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cmpLEQ_cache[hash].vi1 = vi1; 3239ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cmpLEQ_cache[hash].vi2 = vi2; 3240ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cmpLEQ_cache[hash].leq = leq; 3241ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ////-- 3242ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return leq; 3243ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3244ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool VtsID__cmpLEQ ( VtsID vi1, VtsID vi2 ) { 3245ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return LIKELY(vi1 == vi2) ? True : VtsID__cmpLEQ_WRK(vi1, vi2); 3246ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3247ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3248ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* compute binary join */ 3249ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown__attribute__((noinline)) 3250ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic VtsID VtsID__join2_WRK ( VtsID vi1, VtsID vi2 ) { 3251ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt hash; 3252ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID res; 3253b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS *vts1, *vts2; 3254ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //if (vi1 == vi2) return vi1; 3255ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(vi1 != vi2); 3256ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ////++ 3257ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__join2_queries++; 3258ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown hash = hash_VtsIDs(vi1, vi2, N_JOIN2_CACHE); 3259ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (join2_cache[hash].vi1 == vi1 3260ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown && join2_cache[hash].vi2 == vi2) 3261ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return join2_cache[hash].res; 3262ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__join2_misses++; 3263ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ////-- 3264ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts1 = VtsID__to_VTS(vi1); 3265ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts2 = VtsID__to_VTS(vi2); 3266b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov temp_max_sized_VTS->usedTS = 0; 3267b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS__join(temp_max_sized_VTS, vts1,vts2); 3268b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov res = vts_tab__find__or__clone_and_add(temp_max_sized_VTS); 3269ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ////++ 3270ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown join2_cache[hash].vi1 = vi1; 3271ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown join2_cache[hash].vi2 = vi2; 3272ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown join2_cache[hash].res = res; 3273ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ////-- 3274ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return res; 3275ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3276ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline VtsID VtsID__join2 ( VtsID vi1, VtsID vi2 ) { 3277ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return LIKELY(vi1 == vi2) ? vi1 : VtsID__join2_WRK(vi1, vi2); 3278ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3279ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3280ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* create a singleton VTS, namely [thr:1] */ 3281ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic VtsID VtsID__mk_Singleton ( Thr* thr, ULong tym ) { 3282b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov temp_max_sized_VTS->usedTS = 0; 3283b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS__singleton(temp_max_sized_VTS, thr,tym); 3284b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return vts_tab__find__or__clone_and_add(temp_max_sized_VTS); 3285ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3286ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3287ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* tick operation, creates value 1 if specified index is absent */ 3288ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic VtsID VtsID__tick ( VtsID vi, Thr* idx ) { 3289ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VTS* vts = VtsID__to_VTS(vi); 3290b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov temp_max_sized_VTS->usedTS = 0; 3291b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS__tick(temp_max_sized_VTS, idx,vts); 3292b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return vts_tab__find__or__clone_and_add(temp_max_sized_VTS); 3293ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3294ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3295ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* index into a VTS (only for assertions) */ 3296ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic ULong VtsID__indexAt ( VtsID vi, Thr* idx ) { 3297ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VTS* vts = VtsID__to_VTS(vi); 3298ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return VTS__indexAt_SLOW( vts, idx ); 3299ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3300ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3301ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Assuming that !cmpLEQ(vi1, vi2), find the index of the first (or 3302ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown any, really) element in vi1 which is pointwise greater-than the 3303ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown corresponding element in vi2. If no such element exists, return 3304ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown NULL. This needs to be fairly quick since it is called every time 3305ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a race is detected. */ 3306ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Thr* VtsID__findFirst_notLEQ ( VtsID vi1, VtsID vi2 ) 3307ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3308ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VTS *vts1, *vts2; 3309b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Thr* diffthr; 3310b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID diffthrid; 3311ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(vi1 != vi2); 3312ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts1 = VtsID__to_VTS(vi1); 3313ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts2 = VtsID__to_VTS(vi2); 3314ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(vts1 != vts2); 3315b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov diffthrid = VTS__cmpLEQ(vts1, vts2); 3316b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov diffthr = Thr__from_ThrID(diffthrid); 3317ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(diffthr); /* else they are LEQ ! */ 3318ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return diffthr; 3319ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3320ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3321ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3322ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 3323ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 3324ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Filters // 3325ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 3326ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 3327ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3328ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Forget everything we know -- clear the filter and let everything 3329ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown through. This needs to be as fast as possible, since it is called 3330ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown every time the running thread changes, and every time a thread's 3331ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vector clocks change, which can be quite frequent. The obvious 3332ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fast way to do this is simply to stuff in tags which we know are 3333ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown not going to match anything, since they're not aligned to the start 3334ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown of a line. */ 3335436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovstatic void Filter__clear ( Filter* fi, const HChar* who ) 3336ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3337ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i; 3338ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)(" Filter__clear(%p, %s)\n", fi, who); 3339ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < FI_NUM_LINES; i += 8) { 3340ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[i+0] = 1; /* impossible value -- cannot match */ 3341ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[i+1] = 1; 3342ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[i+2] = 1; 3343ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[i+3] = 1; 3344ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[i+4] = 1; 3345ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[i+5] = 1; 3346ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[i+6] = 1; 3347ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[i+7] = 1; 3348ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3349ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(i == FI_NUM_LINES); 3350ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3351ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3352ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Clearing an arbitrary range in the filter. Unfortunately 3353ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown we have to do this due to core-supplied new/die-mem events. */ 3354ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3355ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void Filter__clear_1byte ( Filter* fi, Addr a ) 3356ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3357ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr atag = FI_GET_TAG(a); /* tag of 'a' */ 3358ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord lineno = FI_GET_LINENO(a); /* lineno for 'a' */ 3359ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown FiLine* line = &fi->lines[lineno]; 3360ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord loff = (a - atag) / 8; 3361ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort mask = 0x3 << (2 * (a & 7)); 3362ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* mask is C000, 3000, 0C00, 0300, 00C0, 0030, 000C or 0003 */ 3363ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY( fi->tags[lineno] == atag )) { 3364ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* hit. clear the bits. */ 3365ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort u16 = line->u16s[loff]; 3366ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = u16 & ~mask; /* clear them */ 3367ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 3368ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* miss. The filter doesn't hold this address, so ignore. */ 3369ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3370ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3371ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3372ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void Filter__clear_8bytes_aligned ( Filter* fi, Addr a ) 3373ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3374ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr atag = FI_GET_TAG(a); /* tag of 'a' */ 3375ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord lineno = FI_GET_LINENO(a); /* lineno for 'a' */ 3376ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown FiLine* line = &fi->lines[lineno]; 3377ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord loff = (a - atag) / 8; 3378ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY( fi->tags[lineno] == atag )) { 3379ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = 0; 3380ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 3381ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* miss. The filter doesn't hold this address, so ignore. */ 3382ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3383ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3384ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3385ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void Filter__clear_range ( Filter* fi, Addr a, UWord len ) 3386ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3387ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //VG_(printf)("%lu ", len); 3388ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* slowly do part preceding 8-alignment */ 3389ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (UNLIKELY(!VG_IS_8_ALIGNED(a)) && LIKELY(len > 0)) { 3390ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Filter__clear_1byte( fi, a ); 3391ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a++; 3392ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len--; 3393ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3394ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* vector loop */ 3395ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (len >= 8) { 3396ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Filter__clear_8bytes_aligned( fi, a ); 3397ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 8; 3398ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 8; 3399ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3400ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* slowly do tail */ 3401ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (UNLIKELY(len > 0)) { 3402ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Filter__clear_1byte( fi, a ); 3403ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a++; 3404ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len--; 3405ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3406ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3407ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3408ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3409ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* ------ Read handlers for the filter. ------ */ 3410ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3411ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool Filter__ok_to_skip_crd64 ( Filter* fi, Addr a ) 3412ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3413ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !VG_IS_8_ALIGNED(a) )) 3414ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3415ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown { 3416ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr atag = FI_GET_TAG(a); /* tag of 'a' */ 3417ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord lineno = FI_GET_LINENO(a); /* lineno for 'a' */ 3418ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown FiLine* line = &fi->lines[lineno]; 3419ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord loff = (a - atag) / 8; 3420ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort mask = 0xAAAA; 3421ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY( fi->tags[lineno] == atag )) { 3422ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* hit. check line and update. */ 3423ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort u16 = line->u16s[loff]; 3424ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool ok = (u16 & mask) == mask; /* all R bits set? */ 3425ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = u16 | mask; /* set them */ 3426ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ok; 3427ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 3428ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* miss. nuke existing line and re-use it. */ 3429ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i; 3430ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[lineno] = atag; 3431ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < FI_LINE_SZB / 8; i++) 3432ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[i] = 0; 3433ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = mask; 3434ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3435ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3436ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3437ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3438ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3439ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool Filter__ok_to_skip_crd32 ( Filter* fi, Addr a ) 3440ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3441ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !VG_IS_4_ALIGNED(a) )) 3442ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3443ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown { 3444ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr atag = FI_GET_TAG(a); /* tag of 'a' */ 3445ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord lineno = FI_GET_LINENO(a); /* lineno for 'a' */ 3446ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown FiLine* line = &fi->lines[lineno]; 3447ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord loff = (a - atag) / 8; 3448ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort mask = 0xAA << (2 * (a & 4)); /* 0xAA00 or 0x00AA */ 3449ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY( fi->tags[lineno] == atag )) { 3450ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* hit. check line and update. */ 3451ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort u16 = line->u16s[loff]; 3452ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool ok = (u16 & mask) == mask; /* 4 x R bits set? */ 3453ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = u16 | mask; /* set them */ 3454ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ok; 3455ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 3456ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* miss. nuke existing line and re-use it. */ 3457ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i; 3458ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[lineno] = atag; 3459ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < FI_LINE_SZB / 8; i++) 3460ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[i] = 0; 3461ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = mask; 3462ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3463ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3464ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3465ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3466ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3467ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool Filter__ok_to_skip_crd16 ( Filter* fi, Addr a ) 3468ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3469ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !VG_IS_2_ALIGNED(a) )) 3470ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3471ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown { 3472ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr atag = FI_GET_TAG(a); /* tag of 'a' */ 3473ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord lineno = FI_GET_LINENO(a); /* lineno for 'a' */ 3474ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown FiLine* line = &fi->lines[lineno]; 3475ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord loff = (a - atag) / 8; 3476ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort mask = 0xA << (2 * (a & 6)); 3477ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* mask is A000, 0A00, 00A0 or 000A */ 3478ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY( fi->tags[lineno] == atag )) { 3479ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* hit. check line and update. */ 3480ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort u16 = line->u16s[loff]; 3481ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool ok = (u16 & mask) == mask; /* 2 x R bits set? */ 3482ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = u16 | mask; /* set them */ 3483ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ok; 3484ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 3485ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* miss. nuke existing line and re-use it. */ 3486ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i; 3487ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[lineno] = atag; 3488ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < FI_LINE_SZB / 8; i++) 3489ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[i] = 0; 3490ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = mask; 3491ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3492ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3493ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3494ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3495ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3496ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool Filter__ok_to_skip_crd08 ( Filter* fi, Addr a ) 3497ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3498ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown { 3499ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr atag = FI_GET_TAG(a); /* tag of 'a' */ 3500ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord lineno = FI_GET_LINENO(a); /* lineno for 'a' */ 3501ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown FiLine* line = &fi->lines[lineno]; 3502ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord loff = (a - atag) / 8; 3503ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort mask = 0x2 << (2 * (a & 7)); 3504ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* mask is 8000, 2000, 0800, 0200, 0080, 0020, 0008 or 0002 */ 3505ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY( fi->tags[lineno] == atag )) { 3506ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* hit. check line and update. */ 3507ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort u16 = line->u16s[loff]; 3508ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool ok = (u16 & mask) == mask; /* 1 x R bits set? */ 3509ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = u16 | mask; /* set them */ 3510ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ok; 3511ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 3512ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* miss. nuke existing line and re-use it. */ 3513ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i; 3514ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[lineno] = atag; 3515ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < FI_LINE_SZB / 8; i++) 3516ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[i] = 0; 3517ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = mask; 3518ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3519ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3520ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3521ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3522ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3523ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3524ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* ------ Write handlers for the filter. ------ */ 3525ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3526ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool Filter__ok_to_skip_cwr64 ( Filter* fi, Addr a ) 3527ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3528ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !VG_IS_8_ALIGNED(a) )) 3529ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3530ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown { 3531ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr atag = FI_GET_TAG(a); /* tag of 'a' */ 3532ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord lineno = FI_GET_LINENO(a); /* lineno for 'a' */ 3533ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown FiLine* line = &fi->lines[lineno]; 3534ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord loff = (a - atag) / 8; 3535ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort mask = 0xFFFF; 3536ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY( fi->tags[lineno] == atag )) { 3537ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* hit. check line and update. */ 3538ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort u16 = line->u16s[loff]; 3539ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool ok = (u16 & mask) == mask; /* all R & W bits set? */ 3540ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = u16 | mask; /* set them */ 3541ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ok; 3542ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 3543ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* miss. nuke existing line and re-use it. */ 3544ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i; 3545ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[lineno] = atag; 3546ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < FI_LINE_SZB / 8; i++) 3547ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[i] = 0; 3548ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = mask; 3549ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3550ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3551ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3552ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3553ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3554ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool Filter__ok_to_skip_cwr32 ( Filter* fi, Addr a ) 3555ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3556ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !VG_IS_4_ALIGNED(a) )) 3557ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3558ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown { 3559ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr atag = FI_GET_TAG(a); /* tag of 'a' */ 3560ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord lineno = FI_GET_LINENO(a); /* lineno for 'a' */ 3561ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown FiLine* line = &fi->lines[lineno]; 3562ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord loff = (a - atag) / 8; 3563ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort mask = 0xFF << (2 * (a & 4)); /* 0xFF00 or 0x00FF */ 3564ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY( fi->tags[lineno] == atag )) { 3565ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* hit. check line and update. */ 3566ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort u16 = line->u16s[loff]; 3567ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool ok = (u16 & mask) == mask; /* 4 x R & W bits set? */ 3568ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = u16 | mask; /* set them */ 3569ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ok; 3570ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 3571ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* miss. nuke existing line and re-use it. */ 3572ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i; 3573ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[lineno] = atag; 3574ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < FI_LINE_SZB / 8; i++) 3575ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[i] = 0; 3576ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = mask; 3577ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3578ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3579ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3580ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3581ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3582ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool Filter__ok_to_skip_cwr16 ( Filter* fi, Addr a ) 3583ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3584ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !VG_IS_2_ALIGNED(a) )) 3585ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3586ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown { 3587ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr atag = FI_GET_TAG(a); /* tag of 'a' */ 3588ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord lineno = FI_GET_LINENO(a); /* lineno for 'a' */ 3589ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown FiLine* line = &fi->lines[lineno]; 3590ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord loff = (a - atag) / 8; 3591ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort mask = 0xF << (2 * (a & 6)); 3592ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* mask is F000, 0F00, 00F0 or 000F */ 3593ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY( fi->tags[lineno] == atag )) { 3594ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* hit. check line and update. */ 3595ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort u16 = line->u16s[loff]; 3596ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool ok = (u16 & mask) == mask; /* 2 x R & W bits set? */ 3597ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = u16 | mask; /* set them */ 3598ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ok; 3599ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 3600ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* miss. nuke existing line and re-use it. */ 3601ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i; 3602ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[lineno] = atag; 3603ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < FI_LINE_SZB / 8; i++) 3604ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[i] = 0; 3605ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = mask; 3606ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3607ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3608ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3609ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3610ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3611ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool Filter__ok_to_skip_cwr08 ( Filter* fi, Addr a ) 3612ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3613ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown { 3614ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr atag = FI_GET_TAG(a); /* tag of 'a' */ 3615ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord lineno = FI_GET_LINENO(a); /* lineno for 'a' */ 3616ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown FiLine* line = &fi->lines[lineno]; 3617ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord loff = (a - atag) / 8; 3618ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort mask = 0x3 << (2 * (a & 7)); 3619ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* mask is C000, 3000, 0C00, 0300, 00C0, 0030, 000C or 0003 */ 3620ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY( fi->tags[lineno] == atag )) { 3621ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* hit. check line and update. */ 3622ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort u16 = line->u16s[loff]; 3623ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool ok = (u16 & mask) == mask; /* 1 x R bits set? */ 3624ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = u16 | mask; /* set them */ 3625ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ok; 3626ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 3627ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* miss. nuke existing line and re-use it. */ 3628ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i; 3629ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[lineno] = atag; 3630ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < FI_LINE_SZB / 8; i++) 3631ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[i] = 0; 3632ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = mask; 3633ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3634ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3635ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3636ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3637ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3638ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3639ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 3640ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 3641ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Threads // 3642ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 3643ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 3644ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3645b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Maps ThrID values to their Thr*s (which contain ThrID values that 3646b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov should point back to the relevant slot in the array. Lowest 3647b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov numbered slot (0) is for thrid = 1024, (1) is for 1025, etc. */ 3648b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic XArray* /* of Thr* */ thrid_to_thr_map = NULL; 3649ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3650b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* And a counter to dole out ThrID values. For rationale/background, 3651b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov see comments on definition of ScalarTS (far) above. */ 3652b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic ThrID thrid_counter = 1024; /* runs up to ThrID_MAX_VALID */ 3653ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3654b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic ThrID Thr__to_ThrID ( Thr* thr ) { 3655b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return thr->thrid; 3656b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 3657b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic Thr* Thr__from_ThrID ( UInt thrid ) { 3658b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Thr* thr = *(Thr**)VG_(indexXA)( thrid_to_thr_map, thrid - 1024 ); 3659b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(thr->thrid == thrid); 3660b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return thr; 3661b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 3662ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3663b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic Thr* Thr__new ( void ) 3664b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 3665ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Thr* thr = HG_(zalloc)( "libhb.Thr__new.1", sizeof(Thr) ); 3666ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr->viR = VtsID_INVALID; 3667ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr->viW = VtsID_INVALID; 3668b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thr->llexit_done = False; 3669b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thr->joinedwith_done = False; 3670ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr->filter = HG_(zalloc)( "libhb.Thr__new.2", sizeof(Filter) ); 3671eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov if (HG_(clo_history_level) == 1) 3672eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov thr->local_Kws_n_stacks 3673eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov = VG_(newXA)( HG_(zalloc), 3674eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov "libhb.Thr__new.3 (local_Kws_and_stacks)", 3675eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov HG_(free), sizeof(ULong_n_EC) ); 3676b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3677b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Add this Thr* <-> ThrID binding to the mapping, and 3678b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cross-check */ 3679b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (!thrid_to_thr_map) { 3680b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid_to_thr_map = VG_(newXA)( HG_(zalloc), "libhb.Thr__new.4", 3681b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov HG_(free), sizeof(Thr*) ); 3682b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(thrid_to_thr_map); 3683b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 3684b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3685b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (thrid_counter >= ThrID_MAX_VALID) { 3686b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* We're hosed. We have to stop. */ 3687b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov scalarts_limitations_fail_NORETURN( True/*due_to_nThrs*/ ); 3688b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 3689b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3690b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thr->thrid = thrid_counter++; 3691b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Word ix = VG_(addToXA)( thrid_to_thr_map, &thr ); 3692b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(ix + 1024 == thr->thrid); 3693b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3694ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return thr; 3695ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3696ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3697ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void note_local_Kw_n_stack_for ( Thr* thr ) 3698ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3699ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word nPresent; 3700ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ULong_n_EC pair; 3701ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(thr); 3702ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3703ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // We only collect this info at history level 1 (approx) 3704ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (HG_(clo_history_level) != 1) 3705ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 3706ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3707ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* This is the scalar Kw for thr. */ 3708ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown pair.ull = VtsID__indexAt( thr->viW, thr ); 3709ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown pair.ec = main_get_EC( thr ); 3710ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(pair.ec); 3711ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(thr->local_Kws_n_stacks); 3712ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3713ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* check that we're not adding duplicates */ 3714ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nPresent = VG_(sizeXA)( thr->local_Kws_n_stacks ); 3715ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3716ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Throw away old stacks, if necessary. We can't accumulate stuff 3717ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown indefinitely. */ 3718ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (nPresent >= N_KWs_N_STACKs_PER_THREAD) { 3719ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(dropHeadXA)( thr->local_Kws_n_stacks, nPresent / 2 ); 3720ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nPresent = VG_(sizeXA)( thr->local_Kws_n_stacks ); 3721ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) 3722ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("LOCAL Kw: thr %p, Kw %llu, ec %p (!!! gc !!!)\n", 3723ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr, pair.ull, pair.ec ); 3724ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3725ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3726ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (nPresent > 0) { 3727ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ULong_n_EC* prevPair 3728ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown = (ULong_n_EC*)VG_(indexXA)( thr->local_Kws_n_stacks, nPresent-1 ); 3729ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert( prevPair->ull <= pair.ull ); 3730ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3731ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3732ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (nPresent == 0) 3733ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown pair.ec = NULL; 3734ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3735ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(addToXA)( thr->local_Kws_n_stacks, &pair ); 3736ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3737ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) 3738ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("LOCAL Kw: thr %p, Kw %llu, ec %p\n", 3739ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr, pair.ull, pair.ec ); 3740ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) 3741ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(pp_ExeContext)(pair.ec); 3742ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3743ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3744436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovstatic Int cmp__ULong_n_EC__by_ULong ( const ULong_n_EC* pair1, 3745436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov const ULong_n_EC* pair2 ) 3746ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3747ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (pair1->ull < pair2->ull) return -1; 3748ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (pair1->ull > pair2->ull) return 1; 3749ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0; 3750ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3751ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3752ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3753ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 3754ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 3755ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Shadow Values // 3756ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 3757ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 3758ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3759ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// type SVal, SVal_INVALID and SVal_NOACCESS are defined by 3760ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// hb_zsm.h. We have to do everything else here. 3761ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3762ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* SVal is 64 bit unsigned int. 3763ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3764ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown <---------30---------> <---------30---------> 3765ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 00 X-----Rmin-VtsID-----X 00 X-----Wmin-VtsID-----X C(Rmin,Wmin) 3766ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 10 X--------------------X XX X--------------------X A: SVal_NOACCESS 3767ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 11 0--------------------0 00 0--------------------0 A: SVal_INVALID 3768ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3769ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 3770ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define SVAL_TAGMASK (3ULL << 62) 3771ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3772ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool SVal__isC ( SVal s ) { 3773ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return (0ULL << 62) == (s & SVAL_TAGMASK); 3774ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3775ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline SVal SVal__mkC ( VtsID rmini, VtsID wmini ) { 3776ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //tl_assert(VtsID__is_valid(rmini)); 3777ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //tl_assert(VtsID__is_valid(wmini)); 3778ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return (((ULong)rmini) << 32) | ((ULong)wmini); 3779ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3780ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline VtsID SVal__unC_Rmin ( SVal s ) { 3781ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(SVal__isC(s)); 3782ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return (VtsID)(s >> 32); 3783ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3784ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline VtsID SVal__unC_Wmin ( SVal s ) { 3785ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(SVal__isC(s)); 3786ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return (VtsID)(s & 0xFFFFFFFFULL); 3787ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3788ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3789ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool SVal__isA ( SVal s ) { 3790ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return (2ULL << 62) == (s & SVAL_TAGMASK); 3791ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3792ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline SVal SVal__mkA ( void ) { 3793ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 2ULL << 62; 3794ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3795ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3796ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Direct callback from lib_zsm. */ 3797ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void SVal__rcinc ( SVal s ) { 3798ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (SVal__isC(s)) { 3799ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc( SVal__unC_Rmin(s) ); 3800ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc( SVal__unC_Wmin(s) ); 3801ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3802ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3803ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3804ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Direct callback from lib_zsm. */ 3805ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void SVal__rcdec ( SVal s ) { 3806ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (SVal__isC(s)) { 3807ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcdec( SVal__unC_Rmin(s) ); 3808ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcdec( SVal__unC_Wmin(s) ); 3809ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3810ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3811ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3812ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3813ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 3814ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 3815ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Change-event map2 // 3816ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 3817ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 3818ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3819ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define EVENT_MAP_GC_DISCARD_FRACTION 0.5 3820ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3821ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* This is in two parts: 3822ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3823ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1. A hash table of RCECs. This is a set of reference-counted stack 3824ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown traces. When the reference count of a stack trace becomes zero, 3825ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown it is removed from the set and freed up. The intent is to have 3826ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a set of stack traces which can be referred to from (2), but to 3827ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown only represent each one once. The set is indexed/searched by 3828ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ordering on the stack trace vectors. 3829ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3830ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2. A SparseWA of OldRefs. These store information about each old 3831ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ref that we need to record. It is indexed by address of the 3832ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown location for which the information is recorded. For LRU 3833ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown purposes, each OldRef also contains a generation number, 3834ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown indicating when it was most recently accessed. 3835ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3836ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown The important part of an OldRef is, however, its accs[] array. 3837ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown This is an array of N_OLDREF_ACCS which binds (thread, R/W, 3838ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown size) triples to RCECs. This allows us to collect the last 3839ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown access-traceback by up to N_OLDREF_ACCS different triples for 3840ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown this location. The accs[] array is a MTF-array. If a binding 3841ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown falls off the end, that's too bad -- we will lose info about 3842ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown that triple's access to this location. 3843ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3844ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown When the SparseWA becomes too big, we can throw away the OldRefs 3845ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown whose generation numbers are below some threshold; hence doing 3846ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown approximate LRU discarding. For each discarded OldRef we must 3847ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown of course decrement the reference count on the all RCECs it 3848ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown refers to, in order that entries from (1) eventually get 3849ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown discarded too. 3850ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3851ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown A major improvement in reliability of this mechanism would be to 3852ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown have a dynamically sized OldRef.accs[] array, so no entries ever 3853ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fall off the end. In investigations (Dec 08) it appears that a 3854ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown major cause for the non-availability of conflicting-access traces 3855ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown in race reports is caused by the fixed size of this array. I 3856ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown suspect for most OldRefs, only a few entries are used, but for a 3857ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown minority of cases there is an overflow, leading to info lossage. 3858ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Investigations also suggest this is very workload and scheduling 3859ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sensitive. Therefore a dynamic sizing would be better. 3860ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3861663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng However, dynamic sizing would defeat the use of a PoolAllocator 3862ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for OldRef structures. And that's important for performance. So 3863ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown it's not straightforward to do. 3864ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 3865ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3866ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3867ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__ctxt_rcdec1 = 0; 3868ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__ctxt_rcdec2 = 0; 3869ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__ctxt_rcdec3 = 0; 3870ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__ctxt_rcdec_calls = 0; 3871ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__ctxt_rcdec_discards = 0; 3872ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__ctxt_rcdec1_eq = 0; 3873ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3874ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__ctxt_tab_curr = 0; 3875ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__ctxt_tab_max = 0; 3876ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3877ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__ctxt_tab_qs = 0; 3878ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__ctxt_tab_cmps = 0; 3879ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3880ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3881ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/////////////////////////////////////////////////////// 3882ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//// Part (1): A hash table of RCECs 3883ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/// 3884ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3885ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_FRAMES 8 3886ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3887ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// (UInt) `echo "Reference Counted Execution Context" | md5sum` 3888ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define RCEC_MAGIC 0xab88abb2UL 3889ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3890ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//#define N_RCEC_TAB 98317 /* prime */ 3891ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_RCEC_TAB 196613 /* prime */ 3892ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3893ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef 3894ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct _RCEC { 3895ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord magic; /* sanity check only */ 3896ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct _RCEC* next; 3897ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord rc; 3898ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord rcX; /* used for crosschecking */ 3899ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord frames_hash; /* hash of all the frames */ 3900ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord frames[N_FRAMES]; 3901ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3902ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown RCEC; 3903ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3904ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic RCEC** contextTab = NULL; /* hash table of RCEC*s */ 3905ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3906ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3907ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Gives an arbitrary total order on RCEC .frames fields */ 3908ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Word RCEC__cmp_by_frames ( RCEC* ec1, RCEC* ec2 ) { 3909ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i; 3910ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ec1 && ec1->magic == RCEC_MAGIC); 3911ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ec2 && ec2->magic == RCEC_MAGIC); 3912ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (ec1->frames_hash < ec2->frames_hash) return -1; 3913ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (ec1->frames_hash > ec2->frames_hash) return 1; 3914ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_FRAMES; i++) { 3915ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (ec1->frames[i] < ec2->frames[i]) return -1; 3916ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (ec1->frames[i] > ec2->frames[i]) return 1; 3917ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3918ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0; 3919ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3920ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3921ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3922ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Dec the ref of this RCEC. */ 3923ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void ctxt__rcdec ( RCEC* ec ) 3924ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3925ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_rcdec_calls++; 3926ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ec && ec->magic == RCEC_MAGIC); 3927ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ec->rc > 0); 3928ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ec->rc--; 3929ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3930ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3931ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void ctxt__rcinc ( RCEC* ec ) 3932ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3933ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ec && ec->magic == RCEC_MAGIC); 3934ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ec->rc++; 3935ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3936ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3937ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3938663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng//////////// BEGIN RCEC pool allocator 3939663860b1408516d02ebfcb3a9999a134e6cfb223Ben Chengstatic PoolAlloc* rcec_pool_allocator; 3940ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3941ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic RCEC* alloc_RCEC ( void ) { 3942663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng return VG_(allocEltPA) ( rcec_pool_allocator ); 3943ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3944ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3945ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void free_RCEC ( RCEC* rcec ) { 3946ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(rcec->magic == RCEC_MAGIC); 3947663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng VG_(freeEltPA)( rcec_pool_allocator, rcec ); 3948ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3949663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng//////////// END RCEC pool allocator 3950ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3951ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3952ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Find 'ec' in the RCEC list whose head pointer lives at 'headp' and 3953ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown move it one step closer the the front of the list, so as to make 3954ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown subsequent searches for it cheaper. */ 3955ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void move_RCEC_one_step_forward ( RCEC** headp, RCEC* ec ) 3956ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3957ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown RCEC *ec0, *ec1, *ec2; 3958ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (ec == *headp) 3959ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); /* already at head of list */ 3960ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ec != NULL); 3961ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ec0 = *headp; 3962ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ec1 = NULL; 3963ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ec2 = NULL; 3964ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (True) { 3965ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (ec0 == NULL || ec0 == ec) break; 3966ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ec2 = ec1; 3967ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ec1 = ec0; 3968ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ec0 = ec0->next; 3969ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3970ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ec0 == ec); 3971ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (ec0 != NULL && ec1 != NULL && ec2 != NULL) { 3972ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown RCEC* tmp; 3973ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* ec0 points to ec, ec1 to its predecessor, and ec2 to ec1's 3974ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown predecessor. Swap ec0 and ec1, that is, move ec0 one step 3975ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown closer to the start of the list. */ 3976ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ec2->next == ec1); 3977ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ec1->next == ec0); 3978ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tmp = ec0->next; 3979ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ec2->next = ec0; 3980ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ec0->next = ec1; 3981ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ec1->next = tmp; 3982ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3983ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown else 3984ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (ec0 != NULL && ec1 != NULL && ec2 == NULL) { 3985ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* it's second in the list. */ 3986ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(*headp == ec1); 3987ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ec1->next == ec0); 3988ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ec1->next = ec0->next; 3989ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ec0->next = ec1; 3990ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *headp = ec0; 3991ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3992ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3993ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3994ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3995ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Find the given RCEC in the tree, and return a pointer to it. Or, 3996ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if not present, add the given one to the tree (by making a copy of 3997ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown it, so the caller can immediately deallocate the original) and 3998ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return a pointer to the copy. The caller can safely have 'example' 3999ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown on its stack, since we will always return a pointer to a copy of 4000ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown it, not to the original. Note that the inserted node will have .rc 4001ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown of zero and so the caller must immediatly increment it. */ 4002ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown__attribute__((noinline)) 4003ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic RCEC* ctxt__find_or_add ( RCEC* example ) 4004ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 4005ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord hent; 4006ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown RCEC* copy; 4007ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(example && example->magic == RCEC_MAGIC); 4008ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(example->rc == 0); 4009ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4010ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Search the hash table to see if we already have it. */ 4011ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_tab_qs++; 4012ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown hent = example->frames_hash % N_RCEC_TAB; 4013ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown copy = contextTab[hent]; 4014ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (1) { 4015ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!copy) break; 4016ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(copy->magic == RCEC_MAGIC); 4017ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_tab_cmps++; 4018ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0 == RCEC__cmp_by_frames(copy, example)) break; 4019ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown copy = copy->next; 4020ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4021ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4022ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (copy) { 4023ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(copy != example); 4024ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* optimisation: if it's not at the head of its list, move 1 4025ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown step fwds, to make future searches cheaper */ 4026ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (copy != contextTab[hent]) { 4027ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown move_RCEC_one_step_forward( &contextTab[hent], copy ); 4028ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4029ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 4030ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown copy = alloc_RCEC(); 4031ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(copy != example); 4032ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *copy = *example; 4033ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown copy->next = contextTab[hent]; 4034ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown contextTab[hent] = copy; 4035ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_tab_curr++; 4036ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (stats__ctxt_tab_curr > stats__ctxt_tab_max) 4037ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_tab_max = stats__ctxt_tab_curr; 4038ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4039ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return copy; 4040ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4041ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4042ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline UWord ROLW ( UWord w, Int n ) 4043ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 4044ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Int bpw = 8 * sizeof(UWord); 4045ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown w = (w << n) | (w >> (bpw-n)); 4046ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return w; 4047ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4048ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4049ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown__attribute__((noinline)) 4050ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic RCEC* get_RCEC ( Thr* thr ) 4051ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 4052ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord hash, i; 4053ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown RCEC example; 4054ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown example.magic = RCEC_MAGIC; 4055ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown example.rc = 0; 4056ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown example.rcX = 0; 4057436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov example.next = NULL; 4058ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown main_get_stacktrace( thr, &example.frames[0], N_FRAMES ); 4059ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown hash = 0; 4060ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_FRAMES; i++) { 4061ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown hash ^= example.frames[i]; 4062ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown hash = ROLW(hash, 19); 4063ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4064ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown example.frames_hash = hash; 4065ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ctxt__find_or_add( &example ); 4066ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4067ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4068ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/////////////////////////////////////////////////////// 4069ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//// Part (2): 4070ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/// A SparseWA guest-addr -> OldRef, that refers to (1) 4071ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/// 4072ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4073ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// (UInt) `echo "Old Reference Information" | md5sum` 4074ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define OldRef_MAGIC 0x30b1f075UL 4075ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4076b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Records an access: a thread, a context (size & writeness) and the 4077b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov number of held locks. The size (1,2,4,8) is encoded as 00 = 1, 01 = 4078b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2, 10 = 4, 11 = 8. 4079ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 4080b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovtypedef 4081b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov struct { 4082b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov RCEC* rcec; 4083b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov WordSetID locksHeldW; 4084b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt thrid : SCALARTS_N_THRBITS; 4085b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt szLg2B : 2; 4086b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt isW : 1; 4087b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 4088b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Thr_n_RCEC; 4089ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4090ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_OLDREF_ACCS 5 4091ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4092ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef 4093ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct { 4094ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord magic; /* sanity check only */ 4095ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord gen; /* when most recently accessed */ 4096ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* or free list when not in use */ 4097b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* unused slots in this array have .thrid == 0, which is invalid */ 4098ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Thr_n_RCEC accs[N_OLDREF_ACCS]; 4099ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4100ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown OldRef; 4101ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4102ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4103663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng//////////// BEGIN OldRef pool allocator 4104663860b1408516d02ebfcb3a9999a134e6cfb223Ben Chengstatic PoolAlloc* oldref_pool_allocator; 4105ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4106ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic OldRef* alloc_OldRef ( void ) { 4107663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng return VG_(allocEltPA) ( oldref_pool_allocator ); 4108ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4109ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4110ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void free_OldRef ( OldRef* r ) { 4111ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(r->magic == OldRef_MAGIC); 4112663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng VG_(freeEltPA)( oldref_pool_allocator, r ); 4113ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4114663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng//////////// END OldRef pool allocator 4115ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4116ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4117ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic SparseWA* oldrefTree = NULL; /* SparseWA* OldRef* */ 4118ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord oldrefGen = 0; /* current LRU generation # */ 4119ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord oldrefTreeN = 0; /* # elems in oldrefTree */ 4120ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord oldrefGenIncAt = 0; /* inc gen # when size hits this */ 4121ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4122ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browninline static UInt min_UInt ( UInt a, UInt b ) { 4123ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return a < b ? a : b; 4124ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4125ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4126ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Compare the intervals [a1,a1+n1) and [a2,a2+n2). Return -1 if the 4127ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown first interval is lower, 1 if the first interval is higher, and 0 4128ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if there is any overlap. Redundant paranoia with casting is there 4129ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown following what looked distinctly like a bug in gcc-4.1.2, in which 4130ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown some of the comparisons were done signedly instead of 4131ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown unsignedly. */ 4132ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Copied from exp-ptrcheck/sg_main.c */ 4133ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Word cmp_nonempty_intervals ( Addr a1, SizeT n1, 4134ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr a2, SizeT n2 ) { 4135ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord a1w = (UWord)a1; 4136ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord n1w = (UWord)n1; 4137ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord a2w = (UWord)a2; 4138ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord n2w = (UWord)n2; 4139ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(n1w > 0 && n2w > 0); 4140ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (a1w + n1w <= a2w) return -1L; 4141ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (a2w + n2w <= a1w) return 1L; 4142ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0; 4143ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4144ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4145ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void event_map_bind ( Addr a, SizeT szB, Bool isW, Thr* thr ) 4146ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 4147ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown OldRef* ref; 4148ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown RCEC* rcec; 4149ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i, j; 4150ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord keyW, valW; 4151ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool b; 4152ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4153b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(thr); 4154b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID thrid = thr->thrid; 4155b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(thrid != 0); /* zero is used to denote an empty slot. */ 4156b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 4157b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov WordSetID locksHeldW = thr->hgthread->locksetW; 4158b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 4159ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcec = get_RCEC( thr ); 4160ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ctxt__rcinc(rcec); 4161ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4162b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt szLg2B = 0; 4163ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown switch (szB) { 4164ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* This doesn't look particularly branch-predictor friendly. */ 4165b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov case 1: szLg2B = 0; break; 4166b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov case 2: szLg2B = 1; break; 4167b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov case 4: szLg2B = 2; break; 4168b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov case 8: szLg2B = 3; break; 4169ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown default: tl_assert(0); 4170ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4171ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4172b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Look in the map to see if we already have a record for this 4173b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov address. */ 4174ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b = VG_(lookupSWA)( oldrefTree, &keyW, &valW, a ); 4175ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4176ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (b) { 4177ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4178ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* We already have a record for this address. We now need to 4179b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov see if we have a stack trace pertaining to this (thrid, R/W, 4180ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown size) triple. */ 4181ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(keyW == a); 4182ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ref = (OldRef*)valW; 4183ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ref->magic == OldRef_MAGIC); 4184ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4185ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_OLDREF_ACCS; i++) { 4186b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (ref->accs[i].thrid != thrid) 4187b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov continue; 4188b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (ref->accs[i].szLg2B != szLg2B) 4189ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown continue; 4190b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (ref->accs[i].isW != (UInt)(isW & 1)) 4191ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown continue; 4192ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* else we have a match, so stop looking. */ 4193ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 4194ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4195ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4196ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (i < N_OLDREF_ACCS) { 4197b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* thread 'thr' has an entry at index 'i'. Update its RCEC. */ 4198ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (i > 0) { 4199ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Thr_n_RCEC tmp = ref->accs[i-1]; 4200ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ref->accs[i-1] = ref->accs[i]; 4201ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ref->accs[i] = tmp; 4202ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown i--; 4203ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4204ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (rcec == ref->accs[i].rcec) stats__ctxt_rcdec1_eq++; 4205ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_rcdec1++; 4206b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ctxt__rcdec( ref->accs[i].rcec ); 4207b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(ref->accs[i].thrid == thrid); 4208b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Update the RCEC and the W-held lockset. */ 4209b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[i].rcec = rcec; 4210b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[i].locksHeldW = locksHeldW; 4211ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 4212b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* No entry for this (thread, R/W, size, nWHeld) quad. 4213b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Shuffle all of them down one slot, and put the new entry 4214b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov at the start of the array. */ 4215b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (ref->accs[N_OLDREF_ACCS-1].thrid != 0) { 4216ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* the last slot is in use. We must dec the rc on the 4217ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown associated rcec. */ 4218ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ref->accs[N_OLDREF_ACCS-1].rcec); 4219ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_rcdec2++; 4220ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0 && 0 == (stats__ctxt_rcdec2 & 0xFFF)) 4221ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("QQQQ %lu overflows\n",stats__ctxt_rcdec2); 4222b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ctxt__rcdec( ref->accs[N_OLDREF_ACCS-1].rcec ); 4223ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 4224ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(!ref->accs[N_OLDREF_ACCS-1].rcec); 4225ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4226ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (j = N_OLDREF_ACCS-1; j >= 1; j--) 4227ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ref->accs[j] = ref->accs[j-1]; 4228b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[0].thrid = thrid; 4229b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[0].szLg2B = szLg2B; 4230b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[0].isW = (UInt)(isW & 1); 4231b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[0].locksHeldW = locksHeldW; 4232b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[0].rcec = rcec; 4233b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* thrid==0 is used to signify an empty slot, so we can't 4234b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov add zero thrid (such a ThrID is invalid anyway). */ 4235b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* tl_assert(thrid != 0); */ /* There's a dominating assert above. */ 4236ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4237ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4238ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ref->gen = oldrefGen; 4239ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4240ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 4241ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4242ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* We don't have a record for this address. Create a new one. */ 4243ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (oldrefTreeN >= oldrefGenIncAt) { 4244ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldrefGen++; 4245ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldrefGenIncAt = oldrefTreeN + 50000; 4246ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)("oldrefTree: new gen %lu at size %lu\n", 4247ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldrefGen, oldrefTreeN ); 4248ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4249ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4250ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ref = alloc_OldRef(); 4251ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ref->magic = OldRef_MAGIC; 4252b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->gen = oldrefGen; 4253b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[0].thrid = thrid; 4254b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[0].szLg2B = szLg2B; 4255b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[0].isW = (UInt)(isW & 1); 4256b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[0].locksHeldW = locksHeldW; 4257b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[0].rcec = rcec; 4258b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 4259b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* thrid==0 is used to signify an empty slot, so we can't 4260b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov add zero thrid (such a ThrID is invalid anyway). */ 4261b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* tl_assert(thrid != 0); */ /* There's a dominating assert above. */ 4262b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 4263b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Clear out the rest of the entries */ 4264ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (j = 1; j < N_OLDREF_ACCS; j++) { 4265b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[j].rcec = NULL; 4266b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[j].thrid = 0; 4267b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[j].szLg2B = 0; 4268b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[j].isW = 0; 4269b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[j].locksHeldW = 0; 4270ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4271ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(addToSWA)( oldrefTree, a, (UWord)ref ); 4272ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldrefTreeN++; 4273ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4274ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4275ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4276ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4277ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4278b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Extract info from the conflicting-access machinery. */ 4279ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownBool libhb_event_map_lookup ( /*OUT*/ExeContext** resEC, 4280b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /*OUT*/Thr** resThr, 4281b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /*OUT*/SizeT* resSzB, 4282b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /*OUT*/Bool* resIsW, 4283b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /*OUT*/WordSetID* locksHeldW, 4284ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Thr* thr, Addr a, SizeT szB, Bool isW ) 4285ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 4286ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i, j; 4287ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown OldRef* ref; 4288ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord keyW, valW; 4289ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool b; 4290ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4291b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID cand_thrid; 4292b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov RCEC* cand_rcec; 4293b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Bool cand_isW; 4294b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov SizeT cand_szB; 4295b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov WordSetID cand_locksHeldW; 4296b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Addr cand_a; 4297ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4298ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr toCheck[15]; 4299ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Int nToCheck = 0; 4300ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4301ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(thr); 4302ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(szB == 8 || szB == 4 || szB == 2 || szB == 1); 4303ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4304b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID thrid = thr->thrid; 4305b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 4306ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown toCheck[nToCheck++] = a; 4307ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = -7; i < (Word)szB; i++) { 4308ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (i != 0) 4309ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown toCheck[nToCheck++] = a + i; 4310ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4311ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(nToCheck <= 15); 4312ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4313ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Now see if we can find a suitable matching event for 4314ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown any of the addresses in toCheck[0 .. nToCheck-1]. */ 4315ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (j = 0; j < nToCheck; j++) { 4316ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4317ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cand_a = toCheck[j]; 4318ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // VG_(printf)("test %ld %p\n", j, cand_a); 4319ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4320ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b = VG_(lookupSWA)( oldrefTree, &keyW, &valW, cand_a ); 4321ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!b) 4322ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown continue; 4323ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4324ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ref = (OldRef*)valW; 4325ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(keyW == cand_a); 4326ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ref->magic == OldRef_MAGIC); 4327b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(ref->accs[0].thrid != 0); /* first slot must always be used */ 4328ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4329b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cand_thrid = 0; /* invalid; see comments in event_map_bind */ 4330b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cand_rcec = NULL; 4331b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cand_isW = False; 4332b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cand_szB = 0; 4333b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cand_locksHeldW = 0; /* always valid; see initialise_data_structures() */ 4334ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4335ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_OLDREF_ACCS; i++) { 4336ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Thr_n_RCEC* cand = &ref->accs[i]; 4337b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cand_rcec = cand->rcec; 4338b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cand_thrid = cand->thrid; 4339b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cand_isW = (Bool)cand->isW; 4340b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cand_szB = 1 << cand->szLg2B; 4341b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cand_locksHeldW = cand->locksHeldW; 4342ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4343b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (cand_thrid == 0) 4344ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* This slot isn't in use. Ignore it. */ 4345ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown continue; 4346ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4347b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (cand_thrid == thrid) 4348ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* This is an access by the same thread, but we're only 4349ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown interested in accesses from other threads. Ignore. */ 4350ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown continue; 4351ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4352ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if ((!cand_isW) && (!isW)) 4353ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* We don't want to report a read racing against another 4354ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown read; that's stupid. So in this case move on. */ 4355ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown continue; 4356ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4357ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (cmp_nonempty_intervals(a, szB, cand_a, cand_szB) != 0) 4358ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* No overlap with the access we're asking about. Ignore. */ 4359ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown continue; 4360ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4361ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* We have a match. Stop searching. */ 4362ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 4363ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4364ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4365ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(i >= 0 && i <= N_OLDREF_ACCS); 4366ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4367ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (i < N_OLDREF_ACCS) { 4368ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Int n, maxNFrames; 4369ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* return with success */ 4370b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(cand_thrid); 4371ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(cand_rcec); 4372ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(cand_rcec->magic == RCEC_MAGIC); 4373ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(cand_szB >= 1); 4374ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Count how many non-zero frames we have. */ 4375ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown maxNFrames = min_UInt(N_FRAMES, VG_(clo_backtrace_size)); 4376ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (n = 0; n < maxNFrames; n++) { 4377ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0 == cand_rcec->frames[n]) break; 4378ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4379b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov *resEC = VG_(make_ExeContext_from_StackTrace) 4380b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov (cand_rcec->frames, n); 4381b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov *resThr = Thr__from_ThrID(cand_thrid); 4382b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov *resSzB = cand_szB; 4383b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov *resIsW = cand_isW; 4384b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov *locksHeldW = cand_locksHeldW; 4385ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return True; 4386ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4387ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4388ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* consider next address in toCheck[] */ 4389ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } /* for (j = 0; j < nToCheck; j++) */ 4390ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4391ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* really didn't find anything. */ 4392ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 4393ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4394ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4395ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void event_map_init ( void ) 4396ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 4397ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i; 4398ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4399663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng /* Context (RCEC) pool allocator */ 4400663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng rcec_pool_allocator = VG_(newPA) ( 4401663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng sizeof(RCEC), 4402663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng 1000 /* RCECs per pool */, 4403663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng HG_(zalloc), 4404663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng "libhb.event_map_init.1 (RCEC pools)", 4405663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng HG_(free) 4406663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng ); 4407ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4408ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Context table */ 4409ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(!contextTab); 4410ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown contextTab = HG_(zalloc)( "libhb.event_map_init.2 (context table)", 4411ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown N_RCEC_TAB * sizeof(RCEC*) ); 4412ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(contextTab); 4413ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_RCEC_TAB; i++) 4414ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown contextTab[i] = NULL; 4415ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4416663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng /* Oldref pool allocator */ 4417663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng oldref_pool_allocator = VG_(newPA)( 4418663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng sizeof(OldRef), 4419663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng 1000 /* OldRefs per pool */, 4420663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng HG_(zalloc), 4421663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng "libhb.event_map_init.3 (OldRef pools)", 4422663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng HG_(free) 4423663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng ); 4424ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4425ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Oldref tree */ 4426ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(!oldrefTree); 4427ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldrefTree = VG_(newSWA)( 4428ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(zalloc), 4429ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "libhb.event_map_init.4 (oldref tree)", 4430ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free) 4431ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ); 4432ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(oldrefTree); 4433ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4434ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldrefGen = 0; 4435ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldrefGenIncAt = 0; 4436ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldrefTreeN = 0; 4437ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4438ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4439ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void event_map__check_reference_counts ( Bool before ) 4440ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 4441ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown RCEC* rcec; 4442ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown OldRef* oldref; 4443ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i; 4444ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord nEnts = 0; 4445ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord keyW, valW; 4446ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4447ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Set the 'check' reference counts to zero. Also, optionally 4448ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown check that the real reference counts are non-zero. We allow 4449ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown these to fall to zero before a GC, but the GC must get rid of 4450ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown all those that are zero, hence none should be zero after a 4451ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown GC. */ 4452ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_RCEC_TAB; i++) { 4453ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (rcec = contextTab[i]; rcec; rcec = rcec->next) { 4454ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nEnts++; 4455ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(rcec); 4456ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(rcec->magic == RCEC_MAGIC); 4457ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!before) 4458ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(rcec->rc > 0); 4459ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcec->rcX = 0; 4460ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4461ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4462ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4463ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* check that the stats are sane */ 4464ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(nEnts == stats__ctxt_tab_curr); 4465ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(stats__ctxt_tab_curr <= stats__ctxt_tab_max); 4466ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4467ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* visit all the referencing points, inc check ref counts */ 4468ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(initIterSWA)( oldrefTree ); 4469ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (VG_(nextIterSWA)( oldrefTree, &keyW, &valW )) { 4470ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldref = (OldRef*)valW; 4471ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(oldref->magic == OldRef_MAGIC); 4472ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_OLDREF_ACCS; i++) { 4473b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID aThrID = oldref->accs[i].thrid; 4474b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov RCEC* aRef = oldref->accs[i].rcec; 4475b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (aThrID != 0) { 4476ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aRef); 4477ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aRef->magic == RCEC_MAGIC); 4478ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown aRef->rcX++; 4479ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 4480ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(!aRef); 4481ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4482ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4483ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4484ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4485ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* compare check ref counts with actual */ 4486ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_RCEC_TAB; i++) { 4487ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (rcec = contextTab[i]; rcec; rcec = rcec->next) { 4488ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(rcec->rc == rcec->rcX); 4489ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4490ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4491ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4492ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4493ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown__attribute__((noinline)) 4494ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void event_map_maybe_GC ( void ) 4495ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 4496ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown OldRef* oldref; 4497ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord keyW, valW, retained, maxGen; 4498ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown XArray* refs2del; 4499ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i, j, n2del; 4500ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4501ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord* genMap = NULL; 4502ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord genMap_min = 0; 4503ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord genMap_size = 0; 4504ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4505ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(oldrefTreeN < HG_(clo_conflict_cache_size))) 4506ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 4507ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4508ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) 4509ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("libhb: event_map GC at size %lu\n", oldrefTreeN); 4510ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4511ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Check for sane command line params. Limit values must match 4512ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown those in hg_process_cmd_line_option. */ 4513ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert( HG_(clo_conflict_cache_size) >= 10*1000 ); 4514ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert( HG_(clo_conflict_cache_size) <= 30*1000*1000 ); 4515ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4516ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Check our counting is sane (expensive) */ 4517ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_CEM) 4518ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(oldrefTreeN == VG_(sizeSWA)( oldrefTree )); 4519ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4520ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Check the reference counts (expensive) */ 4521ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_CEM) 4522ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown event_map__check_reference_counts( True/*before*/ ); 4523ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4524ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Compute the distribution of generation values in the ref tree. 4525ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown There are likely only to be a few different generation numbers 4526ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown in the whole tree, but we don't know what they are. Hence use a 4527ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown dynamically resized array of counters. The array is genMap[0 4528ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown .. genMap_size-1], where genMap[0] is the count for the 4529ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown generation number genMap_min, genMap[1] is the count for 4530ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown genMap_min+1, etc. If a new number is seen outside the range 4531ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown [genMap_min .. genMap_min + genMap_size - 1] then the array is 4532ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown copied into a larger array, and genMap_min and genMap_size are 4533ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown adjusted accordingly. */ 4534ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4535ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* genMap :: generation-number -> count-of-nodes-with-that-number */ 4536ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4537ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(initIterSWA)( oldrefTree ); 4538ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while ( VG_(nextIterSWA)( oldrefTree, &keyW, &valW )) { 4539ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4540ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord ea, key; 4541ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldref = (OldRef*)valW; 4542ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown key = oldref->gen; 4543ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4544ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* BEGIN find 'ea', which is the index in genMap holding the 4545ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown count for generation number 'key'. */ 4546ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(genMap == NULL)) { 4547ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* deal with the first key to be seen, so that the following 4548ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cases don't need to handle the complexity of a NULL count 4549ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown array. */ 4550ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown genMap_min = key; 4551ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown genMap_size = 1; 4552ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown genMap = HG_(zalloc)( "libhb.emmG.1a", 4553ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown genMap_size * sizeof(UWord) ); 4554ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ea = 0; 4555ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)("(%lu) case 1 [%lu .. %lu]\n", 4556ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown key, genMap_min, genMap_min+genMap_size- 1 ); 4557ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4558ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown else 4559ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(key >= genMap_min && key < genMap_min + genMap_size)) { 4560ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* this is the expected (almost-always-happens) case: 'key' 4561ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown is already mapped in the array. */ 4562ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ea = key - genMap_min; 4563ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4564ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown else 4565ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (key < genMap_min) { 4566ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* 'key' appears before the start of the current array. 4567ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Extend the current array by allocating a larger one and 4568ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown copying the current one to the upper end of it. */ 4569ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word more; 4570ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord* map2; 4571ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown more = genMap_min - key; 4572ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(more > 0); 4573ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown map2 = HG_(zalloc)( "libhb.emmG.1b", 4574ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (genMap_size + more) * sizeof(UWord) ); 4575ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(memcpy)( &map2[more], genMap, genMap_size * sizeof(UWord) ); 4576ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free)( genMap ); 4577ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown genMap = map2; 4578ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown genMap_size += more; 4579ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown genMap_min -= more; 4580ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ea = 0; 4581ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(genMap_min == key); 4582ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)("(%lu) case 2 [%lu .. %lu]\n", 4583ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown key, genMap_min, genMap_min+genMap_size- 1 ); 4584ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4585ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown else { 4586ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* 'key' appears after the end of the current array. Extend 4587ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown the current array by allocating a larger one and copying 4588ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown the current one to the lower end of it. */ 4589ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word more; 4590ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord* map2; 4591ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(key >= genMap_min + genMap_size); 4592ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown more = key - (genMap_min + genMap_size) + 1; 4593ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(more > 0); 4594ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown map2 = HG_(zalloc)( "libhb.emmG.1c", 4595ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (genMap_size + more) * sizeof(UWord) ); 4596ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(memcpy)( &map2[0], genMap, genMap_size * sizeof(UWord) ); 4597ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free)( genMap ); 4598ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown genMap = map2; 4599ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown genMap_size += more; 4600ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ea = genMap_size - 1;; 4601ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(genMap_min + genMap_size - 1 == key); 4602ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)("(%lu) case 3 [%lu .. %lu]\n", 4603ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown key, genMap_min, genMap_min+genMap_size- 1 ); 4604ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4605ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* END find 'ea' from 'key' */ 4606ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4607ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ea >= 0 && ea < genMap_size); 4608ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* and the whole point of this elaborate computation of 'ea' is .. */ 4609ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown genMap[ea]++; 4610ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4611ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4612ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(genMap); 4613ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(genMap_size > 0); 4614ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4615ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Sanity check what we just computed */ 4616ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown { UWord sum = 0; 4617ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < genMap_size; i++) { 4618ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)(" xxx: gen %ld has %lu\n", 4619ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown i + genMap_min, genMap[i] ); 4620ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sum += genMap[i]; 4621ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4622ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sum == oldrefTreeN); 4623ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4624ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4625ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Figure out how many generations to throw away */ 4626ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown retained = oldrefTreeN; 4627ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown maxGen = 0; 4628ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4629ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < genMap_size; i++) { 4630ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown keyW = i + genMap_min; 4631ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown valW = genMap[i]; 4632ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(keyW > 0); /* can't allow a generation # 0 */ 4633ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)(" XXX: gen %lu has %lu\n", keyW, valW ); 4634ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(keyW >= maxGen); 4635ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(retained >= valW); 4636ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (retained - valW 4637ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown > (UWord)(HG_(clo_conflict_cache_size) 4638ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown * EVENT_MAP_GC_DISCARD_FRACTION)) { 4639ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown retained -= valW; 4640ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown maxGen = keyW; 4641ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 4642ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 4643ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4644ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4645ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4646ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free)(genMap); 4647ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4648ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(retained >= 0 && retained <= oldrefTreeN); 4649ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4650ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Now make up a big list of the oldrefTree entries we want to 4651ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown delete. We can't simultaneously traverse the tree and delete 4652ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stuff from it, so first we need to copy them off somewhere 4653ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown else. (sigh) */ 4654ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown refs2del = VG_(newXA)( HG_(zalloc), "libhb.emmG.2", 4655ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free), sizeof(Addr) ); 4656ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4657ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (retained < oldrefTreeN) { 4658ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4659ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* This is the normal (expected) case. We discard any ref whose 4660ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown generation number <= maxGen. */ 4661ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(initIterSWA)( oldrefTree ); 4662ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (VG_(nextIterSWA)( oldrefTree, &keyW, &valW )) { 4663ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldref = (OldRef*)valW; 4664ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(oldref->magic == OldRef_MAGIC); 4665ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (oldref->gen <= maxGen) { 4666ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(addToXA)( refs2del, &keyW ); 4667ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4668ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4669ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (VG_(clo_stats)) { 4670ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(message)(Vg_DebugMsg, 4671ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "libhb: EvM GC: delete generations %lu and below, " 4672ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "retaining %lu entries\n", 4673ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown maxGen, retained ); 4674ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4675ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4676ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 4677ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4678ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown static UInt rand_seed = 0; /* leave as static */ 4679ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4680ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Degenerate case: there's only one generation in the entire 4681ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree, so we need to have some other way of deciding which 4682ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown refs to throw away. Just throw out half of them randomly. */ 4683ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(retained == oldrefTreeN); 4684ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(initIterSWA)( oldrefTree ); 4685ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (VG_(nextIterSWA)( oldrefTree, &keyW, &valW )) { 4686ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt n; 4687ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldref = (OldRef*)valW; 4688ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(oldref->magic == OldRef_MAGIC); 4689ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown n = VG_(random)( &rand_seed ); 4690ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if ((n & 0xFFF) < 0x800) { 4691ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(addToXA)( refs2del, &keyW ); 4692ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown retained--; 4693ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4694ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4695ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (VG_(clo_stats)) { 4696ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(message)(Vg_DebugMsg, 4697ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "libhb: EvM GC: randomly delete half the entries, " 4698ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "retaining %lu entries\n", 4699ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown retained ); 4700ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4701ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4702ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4703ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4704ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown n2del = VG_(sizeXA)( refs2del ); 4705ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(n2del == (Word)(oldrefTreeN - retained)); 4706ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4707ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)("%s","deleting entries\n"); 4708ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < n2del; i++) { 4709ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool b; 4710ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr ga2del = *(Addr*)VG_(indexXA)( refs2del, i ); 4711ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b = VG_(delFromSWA)( oldrefTree, &keyW, &valW, ga2del ); 4712ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(b); 4713ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(keyW == ga2del); 4714ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldref = (OldRef*)valW; 4715ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (j = 0; j < N_OLDREF_ACCS; j++) { 4716b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID aThrID = oldref->accs[j].thrid; 4717b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov RCEC* aRef = oldref->accs[j].rcec; 4718ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (aRef) { 4719b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(aThrID != 0); 4720ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_rcdec3++; 4721ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ctxt__rcdec( aRef ); 4722ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 4723b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(aThrID == 0); 4724ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4725ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4726ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4727ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown free_OldRef( oldref ); 4728ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4729ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4730ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(deleteXA)( refs2del ); 4731ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4732ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert( VG_(sizeSWA)( oldrefTree ) == retained ); 4733ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4734ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldrefTreeN = retained; 4735ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldrefGenIncAt = oldrefTreeN; /* start new gen right away */ 4736ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4737ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Throw away all RCECs with zero reference counts */ 4738ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_RCEC_TAB; i++) { 4739ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown RCEC** pp = &contextTab[i]; 4740ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown RCEC* p = *pp; 4741ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (p) { 4742ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (p->rc == 0) { 4743ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *pp = p->next; 4744ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown free_RCEC(p); 4745ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown p = *pp; 4746ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(stats__ctxt_tab_curr > 0); 4747ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_tab_curr--; 4748ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 4749ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown pp = &p->next; 4750ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown p = p->next; 4751ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4752ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4753ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4754ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4755ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Check the reference counts (expensive) */ 4756ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_CEM) 4757ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown event_map__check_reference_counts( False/*after*/ ); 4758ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4759ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //if (0) 4760ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //VG_(printf)("XXXX final sizes: oldrefTree %ld, contextTree %ld\n\n", 4761ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // VG_(OSetGen_Size)(oldrefTree), VG_(OSetGen_Size)(contextTree)); 4762ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4763ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4764ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4765ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4766ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 4767ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 4768ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Core MSM // 4769ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 4770ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 4771ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4772ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Logic in msmcread/msmcwrite updated/verified after re-analysis, 19 4773ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Nov 08, and again after [...], 4774ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown June 09. */ 4775ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4776ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic ULong stats__msmcread = 0; 4777ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic ULong stats__msmcread_change = 0; 4778ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic ULong stats__msmcwrite = 0; 4779ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic ULong stats__msmcwrite_change = 0; 4780ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4781ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Some notes on the H1 history mechanism: 4782ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4783ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Transition rules are: 4784ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4785ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown read_{Kr,Kw}(Cr,Cw) = (Cr, Cr `join` Kw) 4786ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_{Kr,Kw}(Cr,Cw) = (Cr `join` Kw, Cr `join` Kw) 4787ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4788ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown After any access by a thread T to a location L, L's constraint pair 4789ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Cr,Cw) has Cw[T] == T's Kw[T], that is, == T's scalar W-clock. 4790ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4791ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown After a race by thread T conflicting with some previous access by 4792ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown some other thread U, for a location with constraint (before 4793ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown processing the later access) (Cr,Cw), then Cw[U] is the segment in 4794ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown which the previously access lies. 4795ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4796ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Hence in record_race_info, we pass in Cfailed and Kfailed, which 4797ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown are compared so as to find out which thread(s) this access 4798ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown conflicts with. Once that is established, we also require the 4799ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown pre-update Cw for the location, so we can index into it for those 4800ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown threads, to get the scalar clock values for the point at which the 4801ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown former accesses were made. (In fact we only bother to do any of 4802ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown this for an arbitrarily chosen one of the conflicting threads, as 4803ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown that's simpler, it avoids flooding the user with vast amounts of 4804ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mostly useless information, and because the program is wrong if it 4805ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown contains any races at all -- so we don't really need to show all 4806ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown conflicting access pairs initially, so long as we only show none if 4807ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown none exist). 4808ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4809ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown --- 4810ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4811ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown That requires the auxiliary proof that 4812ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4813ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Cr `join` Kw)[T] == Kw[T] 4814ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4815ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Why should that be true? Because for any thread T, Kw[T] >= the 4816ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown scalar clock value for T known by any other thread. In other 4817ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown words, because T's value for its own scalar clock is at least as up 4818ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown to date as the value for it known by any other thread (that is true 4819ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for both the R- and W- scalar clocks). Hence no other thread will 4820ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown be able to feed in a value for that element (indirectly via a 4821ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown constraint) which will exceed Kw[T], and hence the join cannot 4822ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cause that particular element to advance. 4823ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 4824ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4825ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown__attribute__((noinline)) 4826ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void record_race_info ( Thr* acc_thr, 4827ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr acc_addr, SizeT szB, Bool isWrite, 4828ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID Cfailed, 4829ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID Kfailed, 4830ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID Cw ) 4831ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 4832ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Call here to report a race. We just hand it onwards to 4833ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(record_error_Race). If that in turn discovers that the 4834ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown error is going to be collected, then, at history_level 2, that 4835ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown queries the conflicting-event map. The alternative would be to 4836ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown query it right here. But that causes a lot of pointless queries 4837ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for errors which will shortly be discarded as duplicates, and 4838ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown can become a performance overhead; so we defer the query until 4839ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown we know the error is not a duplicate. */ 4840ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4841ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Stacks for the bounds of the (or one of the) conflicting 4842ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown segment(s). These are only set at history_level 1. */ 4843ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ExeContext* hist1_seg_start = NULL; 4844ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ExeContext* hist1_seg_end = NULL; 4845ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Thread* hist1_conf_thr = NULL; 4846ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4847ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(acc_thr); 4848b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(acc_thr->hgthread); 4849b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(acc_thr->hgthread->hbthr == acc_thr); 4850ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(HG_(clo_history_level) >= 0 && HG_(clo_history_level) <= 2); 4851ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4852ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (HG_(clo_history_level) == 1) { 4853ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool found; 4854ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word firstIx, lastIx; 4855ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ULong_n_EC key; 4856ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4857ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* At history_level 1, we must round up the relevant stack-pair 4858ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for the conflicting segment right now. This is because 4859ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown deferring it is complex; we can't (easily) put Kfailed and 4860ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Cfailed into the XError and wait for later without 4861ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown getting tied up in difficulties with VtsID reference 4862ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown counting. So just do it now. */ 4863ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Thr* confThr; 4864ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ULong confTym = 0; 4865ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Which thread are we in conflict with? There may be more than 4866ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown one, in which case VtsID__findFirst_notLEQ selects one arbitrarily 4867ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (in fact it's the one with the lowest Thr* value). */ 4868ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown confThr = VtsID__findFirst_notLEQ( Cfailed, Kfailed ); 4869ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* This must exist! since if it was NULL then there's no 4870ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown conflict (semantics of return value of 4871ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__findFirst_notLEQ), and msmc{read,write}, which has 4872ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown called us, just checked exactly this -- that there was in 4873ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fact a race. */ 4874ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(confThr); 4875ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4876ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Get the scalar clock value that the conflicting thread 4877ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown introduced into the constraint. A careful examination of the 4878ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown base machine rules shows that this must be the same as the 4879ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown conflicting thread's scalar clock when it created this 4880ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown constraint. Hence we know the scalar clock of the 4881ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown conflicting thread when the conflicting access was made. */ 4882ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown confTym = VtsID__indexAt( Cfailed, confThr ); 4883ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4884ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Using this scalar clock, index into the conflicting thread's 4885ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown collection of stack traces made each time its vector clock 4886ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (hence its scalar clock) changed. This gives the stack 4887ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown traces at the start and end of the conflicting segment (well, 4888ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown as per comment just above, of one of the conflicting 4889ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown segments, if there are more than one). */ 4890ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown key.ull = confTym; 4891ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown key.ec = NULL; 4892ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* tl_assert(confThr); -- asserted just above */ 4893ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(confThr->local_Kws_n_stacks); 4894ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown firstIx = lastIx = 0; 4895ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown found = VG_(lookupXA_UNSAFE)( 4896ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown confThr->local_Kws_n_stacks, 4897ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown &key, &firstIx, &lastIx, 4898436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov (XACmpFn_t)cmp__ULong_n_EC__by_ULong 4899ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ); 4900ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)("record_race_info %u %u %u confThr %p " 4901ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "confTym %llu found %d (%lu,%lu)\n", 4902ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Cfailed, Kfailed, Cw, 4903ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown confThr, confTym, found, firstIx, lastIx); 4904ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* We can't indefinitely collect stack traces at VTS 4905ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown transitions, since we'd eventually run out of memory. Hence 4906ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown note_local_Kw_n_stack_for will eventually throw away old 4907ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ones, which in turn means we might fail to find index value 4908ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown confTym in the array. */ 4909ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (found) { 4910ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ULong_n_EC *pair_start, *pair_end; 4911ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown pair_start 4912ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown = (ULong_n_EC*)VG_(indexXA)( confThr->local_Kws_n_stacks, lastIx ); 4913ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown hist1_seg_start = pair_start->ec; 4914ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lastIx+1 < VG_(sizeXA)( confThr->local_Kws_n_stacks )) { 4915ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown pair_end 4916ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown = (ULong_n_EC*)VG_(indexXA)( confThr->local_Kws_n_stacks, 4917ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lastIx+1 ); 4918ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* from properties of VG_(lookupXA) and the comparison fn used: */ 4919ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(pair_start->ull < pair_end->ull); 4920ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown hist1_seg_end = pair_end->ec; 4921ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Could do a bit better here. It may be that pair_end 4922ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown doesn't have a stack, but the following entries in the 4923ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown array have the same scalar Kw and to have a stack. So 4924ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown we should search a bit further along the array than 4925ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lastIx+1 if hist1_seg_end is NULL. */ 4926ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 4927b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (!confThr->llexit_done) 4928ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown hist1_seg_end = main_get_EC( confThr ); 4929ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4930ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // seg_start could be NULL iff this is the first stack in the thread 4931ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //if (seg_start) VG_(pp_ExeContext)(seg_start); 4932ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //if (seg_end) VG_(pp_ExeContext)(seg_end); 4933b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov hist1_conf_thr = confThr->hgthread; 4934ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4935ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4936ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4937b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov HG_(record_error_Race)( acc_thr->hgthread, acc_addr, 4938ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown szB, isWrite, 4939ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown hist1_conf_thr, hist1_seg_start, hist1_seg_end ); 4940ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4941ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4942ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Bool is_sane_SVal_C ( SVal sv ) { 4943ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool leq; 4944ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!SVal__isC(sv)) return True; 4945ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown leq = VtsID__cmpLEQ( SVal__unC_Rmin(sv), SVal__unC_Wmin(sv) ); 4946ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return leq; 4947ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4948ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4949ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4950ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Compute new state following a read */ 4951ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline SVal msmcread ( SVal svOld, 4952ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* The following are only needed for 4953ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown creating error reports. */ 4954ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Thr* acc_thr, 4955ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr acc_addr, SizeT szB ) 4956ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 4957ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal svNew = SVal_INVALID; 4958ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__msmcread++; 4959ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4960ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Redundant sanity check on the constraints */ 4961ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_MSM) { 4962ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_SVal_C(svOld)); 4963ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4964ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4965ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(SVal__isC(svOld))) { 4966ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID tviR = acc_thr->viR; 4967ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID tviW = acc_thr->viW; 4968ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID rmini = SVal__unC_Rmin(svOld); 4969ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID wmini = SVal__unC_Wmin(svOld); 4970ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool leq = VtsID__cmpLEQ(rmini,tviR); 4971ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(leq)) { 4972ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* no race */ 4973ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Note: RWLOCK subtlety: use tviW, not tviR */ 4974ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = SVal__mkC( rmini, VtsID__join2(wmini, tviW) ); 4975ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto out; 4976ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 4977ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* assert on sanity of constraints. */ 4978ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool leqxx = VtsID__cmpLEQ(rmini,wmini); 4979ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(leqxx); 4980ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // same as in non-race case 4981ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = SVal__mkC( rmini, VtsID__join2(wmini, tviW) ); 4982ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown record_race_info( acc_thr, acc_addr, szB, False/*!isWrite*/, 4983ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rmini, /* Cfailed */ 4984ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tviR, /* Kfailed */ 4985ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown wmini /* Cw */ ); 4986ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto out; 4987ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4988ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4989ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (SVal__isA(svOld)) { 4990ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* reading no-access memory (sigh); leave unchanged */ 4991ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* check for no pollution */ 4992ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svOld == SVal_NOACCESS); 4993ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = SVal_NOACCESS; 4994ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto out; 4995ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4996ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)("msmcread: bad svOld: 0x%016llx\n", svOld); 4997ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 4998ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4999ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown out: 5000ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_MSM) { 5001ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_SVal_C(svNew)); 5002ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5003ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(svNew != svOld)) { 5004ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5005ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (HG_(clo_history_level) >= 2 5006ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown && SVal__isC(svOld) && SVal__isC(svNew)) { 5007ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown event_map_bind( acc_addr, szB, False/*!isWrite*/, acc_thr ); 5008ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__msmcread_change++; 5009ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5010ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5011ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return svNew; 5012ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5013ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5014ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5015ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Compute new state following a write */ 5016ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline SVal msmcwrite ( SVal svOld, 5017ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* The following are only needed for 5018ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown creating error reports. */ 5019ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Thr* acc_thr, 5020ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr acc_addr, SizeT szB ) 5021ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 5022ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal svNew = SVal_INVALID; 5023ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__msmcwrite++; 5024ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5025ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Redundant sanity check on the constraints */ 5026ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_MSM) { 5027ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_SVal_C(svOld)); 5028ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5029ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5030ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(SVal__isC(svOld))) { 5031ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID tviW = acc_thr->viW; 5032ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID wmini = SVal__unC_Wmin(svOld); 5033ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool leq = VtsID__cmpLEQ(wmini,tviW); 5034ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(leq)) { 5035ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* no race */ 5036ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = SVal__mkC( tviW, tviW ); 5037ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto out; 5038ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 5039ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID rmini = SVal__unC_Rmin(svOld); 5040ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* assert on sanity of constraints. */ 5041ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool leqxx = VtsID__cmpLEQ(rmini,wmini); 5042ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(leqxx); 5043ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // same as in non-race case 5044ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // proof: in the non-race case, we have 5045ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // rmini <= wmini (invar on constraints) 5046ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // tviW <= tviR (invar on thread clocks) 5047ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // wmini <= tviW (from run-time check) 5048ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // hence from transitivity of <= we have 5049ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // rmini <= wmini <= tviW 5050ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // and so join(rmini,tviW) == tviW 5051ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // and join(wmini,tviW) == tviW 5052ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // qed. 5053ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = SVal__mkC( VtsID__join2(rmini, tviW), 5054ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__join2(wmini, tviW) ); 5055ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown record_race_info( acc_thr, acc_addr, szB, True/*isWrite*/, 5056ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown wmini, /* Cfailed */ 5057ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tviW, /* Kfailed */ 5058ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown wmini /* Cw */ ); 5059ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto out; 5060ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5061ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5062ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (SVal__isA(svOld)) { 5063ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* writing no-access memory (sigh); leave unchanged */ 5064ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* check for no pollution */ 5065ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svOld == SVal_NOACCESS); 5066ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = SVal_NOACCESS; 5067ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto out; 5068ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5069ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)("msmcwrite: bad svOld: 0x%016llx\n", svOld); 5070ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 5071ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5072ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown out: 5073ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_MSM) { 5074ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_SVal_C(svNew)); 5075ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5076ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(svNew != svOld)) { 5077ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5078ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (HG_(clo_history_level) >= 2 5079ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown && SVal__isC(svOld) && SVal__isC(svNew)) { 5080ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown event_map_bind( acc_addr, szB, True/*isWrite*/, acc_thr ); 5081ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__msmcwrite_change++; 5082ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5083ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5084ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return svNew; 5085ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5086ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5087ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5088ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 5089ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 5090ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Apply core MSM to specific memory locations // 5091ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 5092ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 5093ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5094ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------- ZSM accesses: 8 bit sapply ------------- */ 5095ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5096ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_sapply08__msmcread ( Thr* thr, Addr a ) { 5097ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 5098ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord cloff, tno, toff; 5099ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal svOld, svNew; 5100ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr; 5101ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cread08s++; 5102ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = get_cacheline(a); 5103ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cloff = get_cacheline_offset(a); 5104ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tno = get_treeno(a); 5105ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown toff = get_tree_offset(a); /* == 0 .. 7 */ 5106ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = cl->descrs[tno]; 5107ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !(descr & (TREE_DESCR_8_0 << toff)) )) { 5108ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal* tree = &cl->svals[tno << 3]; 5109ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = pulldown_to_8(tree, toff, descr); 5110ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5111ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */ 5112ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5113ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svOld = cl->svals[cloff]; 5114ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = msmcread( svOld, thr,a,1 ); 5115ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5116ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5117ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff] = svNew; 5118ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5119ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5120ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_sapply08__msmcwrite ( Thr* thr, Addr a ) { 5121ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 5122ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord cloff, tno, toff; 5123ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal svOld, svNew; 5124ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr; 5125ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cwrite08s++; 5126ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = get_cacheline(a); 5127ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cloff = get_cacheline_offset(a); 5128ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tno = get_treeno(a); 5129ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown toff = get_tree_offset(a); /* == 0 .. 7 */ 5130ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = cl->descrs[tno]; 5131ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !(descr & (TREE_DESCR_8_0 << toff)) )) { 5132ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal* tree = &cl->svals[tno << 3]; 5133ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = pulldown_to_8(tree, toff, descr); 5134ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5135ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */ 5136ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5137ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svOld = cl->svals[cloff]; 5138ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = msmcwrite( svOld, thr,a,1 ); 5139ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5140ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5141ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff] = svNew; 5142ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5143ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5144ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------- ZSM accesses: 16 bit sapply ------------- */ 5145ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5146ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_sapply16__msmcread ( Thr* thr, Addr a ) { 5147ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 5148ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord cloff, tno, toff; 5149ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal svOld, svNew; 5150ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr; 5151ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cread16s++; 5152ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(!aligned16(a))) goto slowcase; 5153ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = get_cacheline(a); 5154ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cloff = get_cacheline_offset(a); 5155ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tno = get_treeno(a); 5156ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown toff = get_tree_offset(a); /* == 0, 2, 4 or 6 */ 5157ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = cl->descrs[tno]; 5158ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !(descr & (TREE_DESCR_16_0 << toff)) )) { 5159ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (valid_value_is_below_me_16(descr, toff)) { 5160ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto slowcase; 5161ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 5162ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal* tree = &cl->svals[tno << 3]; 5163ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = pulldown_to_16(tree, toff, descr); 5164ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5165ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5166ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */ 5167ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5168ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svOld = cl->svals[cloff]; 5169ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = msmcread( svOld, thr,a,2 ); 5170ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5171ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5172ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff] = svNew; 5173ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5174ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown slowcase: /* misaligned, or must go further down the tree */ 5175ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_16to8splits++; 5176ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply08__msmcread( thr, a + 0 ); 5177ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply08__msmcread( thr, a + 1 ); 5178ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5179ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5180ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_sapply16__msmcwrite ( Thr* thr, Addr a ) { 5181ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 5182ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord cloff, tno, toff; 5183ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal svOld, svNew; 5184ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr; 5185ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cwrite16s++; 5186ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(!aligned16(a))) goto slowcase; 5187ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = get_cacheline(a); 5188ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cloff = get_cacheline_offset(a); 5189ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tno = get_treeno(a); 5190ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown toff = get_tree_offset(a); /* == 0, 2, 4 or 6 */ 5191ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = cl->descrs[tno]; 5192ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !(descr & (TREE_DESCR_16_0 << toff)) )) { 5193ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (valid_value_is_below_me_16(descr, toff)) { 5194ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto slowcase; 5195ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 5196ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal* tree = &cl->svals[tno << 3]; 5197ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = pulldown_to_16(tree, toff, descr); 5198ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5199ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5200ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */ 5201ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5202ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svOld = cl->svals[cloff]; 5203ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = msmcwrite( svOld, thr,a,2 ); 5204ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5205ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5206ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff] = svNew; 5207ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5208ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown slowcase: /* misaligned, or must go further down the tree */ 5209ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_16to8splits++; 5210ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply08__msmcwrite( thr, a + 0 ); 5211ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply08__msmcwrite( thr, a + 1 ); 5212ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5213ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5214ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------- ZSM accesses: 32 bit sapply ------------- */ 5215ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5216ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_sapply32__msmcread ( Thr* thr, Addr a ) { 5217ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 5218ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord cloff, tno, toff; 5219ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal svOld, svNew; 5220ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr; 5221ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cread32s++; 5222ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(!aligned32(a))) goto slowcase; 5223ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = get_cacheline(a); 5224ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cloff = get_cacheline_offset(a); 5225ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tno = get_treeno(a); 5226ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown toff = get_tree_offset(a); /* == 0 or 4 */ 5227ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = cl->descrs[tno]; 5228ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !(descr & (TREE_DESCR_32_0 << toff)) )) { 5229ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (valid_value_is_above_me_32(descr, toff)) { 5230ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal* tree = &cl->svals[tno << 3]; 5231ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = pulldown_to_32(tree, toff, descr); 5232ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 5233ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto slowcase; 5234ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5235ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5236ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */ 5237ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5238ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svOld = cl->svals[cloff]; 5239ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = msmcread( svOld, thr,a,4 ); 5240ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5241ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5242ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff] = svNew; 5243ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5244ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown slowcase: /* misaligned, or must go further down the tree */ 5245ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_32to16splits++; 5246ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply16__msmcread( thr, a + 0 ); 5247ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply16__msmcread( thr, a + 2 ); 5248ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5249ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5250ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_sapply32__msmcwrite ( Thr* thr, Addr a ) { 5251ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 5252ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord cloff, tno, toff; 5253ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal svOld, svNew; 5254ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr; 5255ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cwrite32s++; 5256ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(!aligned32(a))) goto slowcase; 5257ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = get_cacheline(a); 5258ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cloff = get_cacheline_offset(a); 5259ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tno = get_treeno(a); 5260ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown toff = get_tree_offset(a); /* == 0 or 4 */ 5261ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = cl->descrs[tno]; 5262ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !(descr & (TREE_DESCR_32_0 << toff)) )) { 5263ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (valid_value_is_above_me_32(descr, toff)) { 5264ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal* tree = &cl->svals[tno << 3]; 5265ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = pulldown_to_32(tree, toff, descr); 5266ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 5267ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto slowcase; 5268ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5269ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5270ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */ 5271ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5272ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svOld = cl->svals[cloff]; 5273ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = msmcwrite( svOld, thr,a,4 ); 5274ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5275ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5276ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff] = svNew; 5277ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5278ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown slowcase: /* misaligned, or must go further down the tree */ 5279ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_32to16splits++; 5280ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply16__msmcwrite( thr, a + 0 ); 5281ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply16__msmcwrite( thr, a + 2 ); 5282ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5283ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5284ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------- ZSM accesses: 64 bit sapply ------------- */ 5285ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5286ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_sapply64__msmcread ( Thr* thr, Addr a ) { 5287ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 5288ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord cloff, tno; 5289ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //UWord toff; 5290ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal svOld, svNew; 5291ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr; 5292ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cread64s++; 5293ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(!aligned64(a))) goto slowcase; 5294ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = get_cacheline(a); 5295ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cloff = get_cacheline_offset(a); 5296ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tno = get_treeno(a); 5297ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //toff = get_tree_offset(a); /* == 0, unused */ 5298ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = cl->descrs[tno]; 5299ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !(descr & TREE_DESCR_64) )) { 5300ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto slowcase; 5301ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5302ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svOld = cl->svals[cloff]; 5303ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = msmcread( svOld, thr,a,8 ); 5304ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5305ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5306ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff] = svNew; 5307ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5308ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown slowcase: /* misaligned, or must go further down the tree */ 5309ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_64to32splits++; 5310ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply32__msmcread( thr, a + 0 ); 5311ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply32__msmcread( thr, a + 4 ); 5312ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5313ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5314ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_sapply64__msmcwrite ( Thr* thr, Addr a ) { 5315ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 5316ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord cloff, tno; 5317ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //UWord toff; 5318ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal svOld, svNew; 5319ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr; 5320ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cwrite64s++; 5321ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(!aligned64(a))) goto slowcase; 5322ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = get_cacheline(a); 5323ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cloff = get_cacheline_offset(a); 5324ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tno = get_treeno(a); 5325ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //toff = get_tree_offset(a); /* == 0, unused */ 5326ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = cl->descrs[tno]; 5327ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !(descr & TREE_DESCR_64) )) { 5328ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto slowcase; 5329ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5330ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svOld = cl->svals[cloff]; 5331ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = msmcwrite( svOld, thr,a,8 ); 5332ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5333ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5334ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff] = svNew; 5335ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5336ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown slowcase: /* misaligned, or must go further down the tree */ 5337ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_64to32splits++; 5338ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply32__msmcwrite( thr, a + 0 ); 5339ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply32__msmcwrite( thr, a + 4 ); 5340ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5341ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5342ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--------------- ZSM accesses: 8 bit swrite --------------- */ 5343ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5344ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 5345ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_swrite08 ( Addr a, SVal svNew ) { 5346ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 5347ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord cloff, tno, toff; 5348ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr; 5349ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_swrite08s++; 5350ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = get_cacheline(a); 5351ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cloff = get_cacheline_offset(a); 5352ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tno = get_treeno(a); 5353ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown toff = get_tree_offset(a); /* == 0 .. 7 */ 5354ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = cl->descrs[tno]; 5355ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !(descr & (TREE_DESCR_8_0 << toff)) )) { 5356ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal* tree = &cl->svals[tno << 3]; 5357ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = pulldown_to_8(tree, toff, descr); 5358ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5359ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */ 5360ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5361ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5362ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff] = svNew; 5363ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5364ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5365ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--------------- ZSM accesses: 16 bit swrite --------------- */ 5366ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5367ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 5368ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_swrite16 ( Addr a, SVal svNew ) { 5369ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 5370ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord cloff, tno, toff; 5371ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr; 5372ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_swrite16s++; 5373ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(!aligned16(a))) goto slowcase; 5374ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = get_cacheline(a); 5375ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cloff = get_cacheline_offset(a); 5376ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tno = get_treeno(a); 5377ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown toff = get_tree_offset(a); /* == 0, 2, 4 or 6 */ 5378ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = cl->descrs[tno]; 5379ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !(descr & (TREE_DESCR_16_0 << toff)) )) { 5380ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (valid_value_is_below_me_16(descr, toff)) { 5381ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Writing at this level. Need to fix up 'descr'. */ 5382ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = pullup_descr_to_16(descr, toff); 5383ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* At this point, the tree does not match cl->descr[tno] any 5384ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown more. The assignments below will fix it up. */ 5385ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 5386ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* We can't indiscriminately write on the w16 node as in the 5387ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown w64 case, as that might make the node inconsistent with 5388ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown its parent. So first, pull down to this level. */ 5389ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal* tree = &cl->svals[tno << 3]; 5390ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = pulldown_to_16(tree, toff, descr); 5391ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5392ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */ 5393ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5394ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5395ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5396ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 0] = svNew; 5397ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 1] = SVal_INVALID; 5398ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5399ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown slowcase: /* misaligned */ 5400ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_16to8splits++; 5401ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite08( a + 0, svNew ); 5402ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite08( a + 1, svNew ); 5403ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5404ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5405ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--------------- ZSM accesses: 32 bit swrite --------------- */ 5406ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5407ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 5408ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_swrite32 ( Addr a, SVal svNew ) { 5409ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 5410ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord cloff, tno, toff; 5411ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr; 5412ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_swrite32s++; 5413ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(!aligned32(a))) goto slowcase; 5414ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = get_cacheline(a); 5415ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cloff = get_cacheline_offset(a); 5416ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tno = get_treeno(a); 5417ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown toff = get_tree_offset(a); /* == 0 or 4 */ 5418ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = cl->descrs[tno]; 5419ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !(descr & (TREE_DESCR_32_0 << toff)) )) { 5420ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (valid_value_is_above_me_32(descr, toff)) { 5421ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* We can't indiscriminately write on the w32 node as in the 5422ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown w64 case, as that might make the node inconsistent with 5423ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown its parent. So first, pull down to this level. */ 5424ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal* tree = &cl->svals[tno << 3]; 5425ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = pulldown_to_32(tree, toff, descr); 5426ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5427ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */ 5428ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 5429ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Writing at this level. Need to fix up 'descr'. */ 5430ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = pullup_descr_to_32(descr, toff); 5431ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* At this point, the tree does not match cl->descr[tno] any 5432ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown more. The assignments below will fix it up. */ 5433ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5434ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5435ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5436ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 0] = svNew; 5437ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 1] = SVal_INVALID; 5438ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 2] = SVal_INVALID; 5439ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 3] = SVal_INVALID; 5440ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5441ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown slowcase: /* misaligned */ 5442ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_32to16splits++; 5443ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite16( a + 0, svNew ); 5444ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite16( a + 2, svNew ); 5445ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5446ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5447ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--------------- ZSM accesses: 64 bit swrite --------------- */ 5448ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5449ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 5450ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_swrite64 ( Addr a, SVal svNew ) { 5451ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 5452ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord cloff, tno; 5453ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //UWord toff; 5454ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_swrite64s++; 5455ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(!aligned64(a))) goto slowcase; 5456ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = get_cacheline(a); 5457ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cloff = get_cacheline_offset(a); 5458ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tno = get_treeno(a); 5459ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //toff = get_tree_offset(a); /* == 0, unused */ 5460ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = TREE_DESCR_64; 5461ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5462ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 0] = svNew; 5463ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 1] = SVal_INVALID; 5464ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 2] = SVal_INVALID; 5465ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 3] = SVal_INVALID; 5466ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 4] = SVal_INVALID; 5467ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 5] = SVal_INVALID; 5468ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 6] = SVal_INVALID; 5469ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 7] = SVal_INVALID; 5470ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5471ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown slowcase: /* misaligned */ 5472ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_64to32splits++; 5473ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite32( a + 0, svNew ); 5474ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite32( a + 4, svNew ); 5475ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5476ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5477ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------- ZSM accesses: 8 bit sread/scopy ------------- */ 5478ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5479ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 5480ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownSVal zsm_sread08 ( Addr a ) { 5481ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 5482ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord cloff, tno, toff; 5483ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr; 5484ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_sread08s++; 5485ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = get_cacheline(a); 5486ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cloff = get_cacheline_offset(a); 5487ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tno = get_treeno(a); 5488ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown toff = get_tree_offset(a); /* == 0 .. 7 */ 5489ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = cl->descrs[tno]; 5490ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !(descr & (TREE_DESCR_8_0 << toff)) )) { 5491ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal* tree = &cl->svals[tno << 3]; 5492ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = pulldown_to_8(tree, toff, descr); 5493ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5494ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return cl->svals[cloff]; 5495ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5496ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5497ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_scopy08 ( Addr src, Addr dst, Bool uu_normalise ) { 5498ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal sv; 5499ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_scopy08s++; 5500ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sv = zsm_sread08( src ); 5501ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite08( dst, sv ); 5502ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5503ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5504ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5505ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Block-copy states (needed for implementing realloc()). Note this 5506ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown doesn't change the filtering arrangements. The caller of 5507ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_scopy_range needs to attend to that. */ 5508ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5509ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_scopy_range ( Addr src, Addr dst, SizeT len ) 5510ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 5511ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT i; 5512ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) 5513ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5514ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5515ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* assert for non-overlappingness */ 5516ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(src+len <= dst || dst+len <= src); 5517ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5518ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* To be simple, just copy byte by byte. But so as not to wreck 5519ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown performance for later accesses to dst[0 .. len-1], normalise 5520ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown destination lines as we finish with them, and also normalise the 5521ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line containing the first and last address. */ 5522ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < len; i++) { 5523ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool normalise 5524ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown = get_cacheline_offset( dst+i+1 ) == 0 /* last in line */ 5525ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown || i == 0 /* first in range */ 5526ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown || i == len-1; /* last in range */ 5527ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_scopy08( src+i, dst+i, normalise ); 5528ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5529ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5530ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5531ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5532ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* For setting address ranges to a given value. Has considerable 5533ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sophistication so as to avoid generating large numbers of pointless 5534ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cache loads/writebacks for large ranges. */ 5535ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5536ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Do small ranges in-cache, in the obvious way. */ 5537ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 5538ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_sset_range_SMALL ( Addr a, SizeT len, SVal svNew ) 5539ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 5540ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* fast track a couple of common cases */ 5541ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 4 && aligned32(a)) { 5542ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite32( a, svNew ); 5543ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5544ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5545ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 8 && aligned64(a)) { 5546ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite64( a, svNew ); 5547ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5548ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5549ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5550ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* be completely general (but as efficient as possible) */ 5551ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5552ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5553ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!aligned16(a) && len >= 1) { 5554ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite08( a, svNew ); 5555ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 1; 5556ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 1; 5557ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned16(a)); 5558ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5559ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5560ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5561ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!aligned32(a) && len >= 2) { 5562ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite16( a, svNew ); 5563ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 2; 5564ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 2; 5565ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned32(a)); 5566ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5567ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5568ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5569ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!aligned64(a) && len >= 4) { 5570ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite32( a, svNew ); 5571ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 4; 5572ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 4; 5573ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned64(a)); 5574ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5575ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5576ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5577ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 8) { 5578ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned64(a)); 5579ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (len >= 8) { 5580ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite64( a, svNew ); 5581ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 8; 5582ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 8; 5583ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5584ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned64(a)); 5585ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5586ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5587ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5588ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 4) 5589ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned32(a)); 5590ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 4) { 5591ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite32( a, svNew ); 5592ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 4; 5593ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 4; 5594ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5595ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5596ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5597ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 2) 5598ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned16(a)); 5599ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 2) { 5600ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite16( a, svNew ); 5601ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 2; 5602ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 2; 5603ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5604ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5605ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5606ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 1) { 5607ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite08( a, svNew ); 5608ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //a += 1; 5609ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 1; 5610ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5611ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(len == 0); 5612ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5613ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5614ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5615ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* If we're doing a small range, hand off to zsm_sset_range_SMALL. But 5616ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for larger ranges, try to operate directly on the out-of-cache 5617ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown representation, rather than dragging lines into the cache, 5618ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown overwriting them, and forcing them out. This turns out to be an 5619ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown important performance optimisation. 5620ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5621ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Note that this doesn't change the filtering arrangements. The 5622ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown caller of zsm_sset_range needs to attend to that. */ 5623ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5624ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_sset_range ( Addr a, SizeT len, SVal svNew ) 5625ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 5626ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5627ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_make_New_arange += (ULong)len; 5628ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5629ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0 && len > 500) 5630ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("make New ( %#lx, %ld )\n", a, len ); 5631ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5632ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) { 5633ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown static UWord n_New_in_cache = 0; 5634ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown static UWord n_New_not_in_cache = 0; 5635ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* tag is 'a' with the in-line offset masked out, 5636ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown eg a[31]..a[4] 0000 */ 5637ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr tag = a & ~(N_LINE_ARANGE - 1); 5638ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord wix = (a >> N_LINE_BITS) & (N_WAY_NENT - 1); 5639ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(tag == cache_shmem.tags0[wix])) { 5640ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown n_New_in_cache++; 5641ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 5642ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown n_New_not_in_cache++; 5643ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5644ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0 == ((n_New_in_cache + n_New_not_in_cache) % 100000)) 5645ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("shadow_mem_make_New: IN %lu OUT %lu\n", 5646ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown n_New_in_cache, n_New_not_in_cache ); 5647ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5648ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5649ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(len < 2 * N_LINE_ARANGE)) { 5650ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sset_range_SMALL( a, len, svNew ); 5651ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 5652ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr before_start = a; 5653ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr aligned_start = cacheline_ROUNDUP(a); 5654ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr after_start = cacheline_ROUNDDN(a + len); 5655ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord before_len = aligned_start - before_start; 5656ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord aligned_len = after_start - aligned_start; 5657ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord after_len = a + len - after_start; 5658ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(before_start <= aligned_start); 5659ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned_start <= after_start); 5660ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(before_len < N_LINE_ARANGE); 5661ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(after_len < N_LINE_ARANGE); 5662ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(get_cacheline_offset(aligned_start) == 0); 5663ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (get_cacheline_offset(a) == 0) { 5664ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(before_len == 0); 5665ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(a == aligned_start); 5666ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5667ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (get_cacheline_offset(a+len) == 0) { 5668ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(after_len == 0); 5669ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(after_start == a+len); 5670ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5671ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (before_len > 0) { 5672ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sset_range_SMALL( before_start, before_len, svNew ); 5673ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5674ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (after_len > 0) { 5675ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sset_range_SMALL( after_start, after_len, svNew ); 5676ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5677ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_make_New_inZrep += (ULong)aligned_len; 5678ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5679ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (1) { 5680ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr tag; 5681ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord wix; 5682ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (aligned_start >= after_start) 5683ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 5684ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(get_cacheline_offset(aligned_start) == 0); 5685ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tag = aligned_start & ~(N_LINE_ARANGE - 1); 5686ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown wix = (aligned_start >> N_LINE_BITS) & (N_WAY_NENT - 1); 5687ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tag == cache_shmem.tags0[wix]) { 5688ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i; 5689ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_LINE_ARANGE / 8; i++) 5690ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite64( aligned_start + i * 8, svNew ); 5691ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 5692ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i; 5693ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word zix; 5694ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SecMap* sm; 5695ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineZ* lineZ; 5696ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* This line is not in the cache. Do not force it in; instead 5697ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown modify it in-place. */ 5698ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* find the Z line to write in and rcdec it or the 5699ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown associated F line. */ 5700ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown find_Z_for_writing( &sm, &zix, tag ); 5701ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm); 5702ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(zix >= 0 && zix < N_SECMAP_ZLINES); 5703ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineZ = &sm->linesZ[zix]; 5704ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineZ->dict[0] = svNew; 5705ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineZ->dict[1] = lineZ->dict[2] = lineZ->dict[3] = SVal_INVALID; 5706ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_LINE_ARANGE/4; i++) 5707ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineZ->ix2s[i] = 0; /* all refer to dict[0] */ 5708ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcinc_LineZ(lineZ); 5709ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5710ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown aligned_start += N_LINE_ARANGE; 5711ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown aligned_len -= N_LINE_ARANGE; 5712ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5713ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned_start == after_start); 5714ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned_len == 0); 5715ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5716ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5717ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5718ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5719ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 5720ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 5721ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Front-filtering accesses // 5722ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 5723ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 5724ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5725ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__f_ac = 0; 5726ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__f_sk = 0; 5727ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5728ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#if 0 5729ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# define STATS__F_SHOW \ 5730ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown do { \ 5731ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(0 == (stats__f_ac & 0xFFFFFF))) \ 5732ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("filters: ac %lu sk %lu\n", \ 5733ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_ac, stats__f_sk); \ 5734ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } while (0) 5735ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#else 5736ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# define STATS__F_SHOW /* */ 5737ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#endif 5738ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5739ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_sapply08_f__msmcwrite ( Thr* thr, Addr a ) { 5740ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_ac++; 5741ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown STATS__F_SHOW; 5742ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(Filter__ok_to_skip_cwr08(thr->filter, a))) { 5743ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_sk++; 5744ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5745ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5746ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply08__msmcwrite(thr, a); 5747ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5748ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5749ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_sapply16_f__msmcwrite ( Thr* thr, Addr a ) { 5750ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_ac++; 5751ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown STATS__F_SHOW; 5752ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(Filter__ok_to_skip_cwr16(thr->filter, a))) { 5753ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_sk++; 5754ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5755ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5756ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply16__msmcwrite(thr, a); 5757ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5758ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5759ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_sapply32_f__msmcwrite ( Thr* thr, Addr a ) { 5760ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_ac++; 5761ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown STATS__F_SHOW; 5762ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(Filter__ok_to_skip_cwr32(thr->filter, a))) { 5763ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_sk++; 5764ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5765ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5766ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply32__msmcwrite(thr, a); 5767ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5768ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5769ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_sapply64_f__msmcwrite ( Thr* thr, Addr a ) { 5770ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_ac++; 5771ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown STATS__F_SHOW; 5772ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(Filter__ok_to_skip_cwr64(thr->filter, a))) { 5773ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_sk++; 5774ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5775ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5776ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply64__msmcwrite(thr, a); 5777ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5778ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5779ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_sapplyNN_f__msmcwrite ( Thr* thr, Addr a, SizeT len ) 5780ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 5781ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* fast track a couple of common cases */ 5782ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 4 && aligned32(a)) { 5783ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply32_f__msmcwrite( thr, a ); 5784ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5785ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5786ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 8 && aligned64(a)) { 5787ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply64_f__msmcwrite( thr, a ); 5788ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5789ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5790ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5791ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* be completely general (but as efficient as possible) */ 5792ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5793ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5794ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!aligned16(a) && len >= 1) { 5795ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply08_f__msmcwrite( thr, a ); 5796ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 1; 5797ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 1; 5798ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned16(a)); 5799ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5800ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5801ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5802ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!aligned32(a) && len >= 2) { 5803ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply16_f__msmcwrite( thr, a ); 5804ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 2; 5805ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 2; 5806ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned32(a)); 5807ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5808ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5809ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5810ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!aligned64(a) && len >= 4) { 5811ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply32_f__msmcwrite( thr, a ); 5812ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 4; 5813ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 4; 5814ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned64(a)); 5815ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5816ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5817ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5818ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 8) { 5819ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned64(a)); 5820ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (len >= 8) { 5821ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply64_f__msmcwrite( thr, a ); 5822ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 8; 5823ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 8; 5824ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5825ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned64(a)); 5826ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5827ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5828ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5829ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 4) 5830ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned32(a)); 5831ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 4) { 5832ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply32_f__msmcwrite( thr, a ); 5833ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 4; 5834ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 4; 5835ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5836ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5837ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5838ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 2) 5839ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned16(a)); 5840ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 2) { 5841ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply16_f__msmcwrite( thr, a ); 5842ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 2; 5843ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 2; 5844ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5845ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5846ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5847ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 1) { 5848ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply08_f__msmcwrite( thr, a ); 5849ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //a += 1; 5850ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 1; 5851ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5852ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(len == 0); 5853ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5854ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5855ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_sapply08_f__msmcread ( Thr* thr, Addr a ) { 5856ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_ac++; 5857ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown STATS__F_SHOW; 5858ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(Filter__ok_to_skip_crd08(thr->filter, a))) { 5859ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_sk++; 5860ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5861ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5862ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply08__msmcread(thr, a); 5863ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5864ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5865ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_sapply16_f__msmcread ( Thr* thr, Addr a ) { 5866ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_ac++; 5867ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown STATS__F_SHOW; 5868ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(Filter__ok_to_skip_crd16(thr->filter, a))) { 5869ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_sk++; 5870ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5871ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5872ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply16__msmcread(thr, a); 5873ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5874ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5875ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_sapply32_f__msmcread ( Thr* thr, Addr a ) { 5876ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_ac++; 5877ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown STATS__F_SHOW; 5878ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(Filter__ok_to_skip_crd32(thr->filter, a))) { 5879ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_sk++; 5880ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5881ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5882ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply32__msmcread(thr, a); 5883ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5884ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5885ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_sapply64_f__msmcread ( Thr* thr, Addr a ) { 5886ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_ac++; 5887ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown STATS__F_SHOW; 5888ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(Filter__ok_to_skip_crd64(thr->filter, a))) { 5889ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_sk++; 5890ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5891ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5892ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply64__msmcread(thr, a); 5893ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5894ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5895ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_sapplyNN_f__msmcread ( Thr* thr, Addr a, SizeT len ) 5896ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 5897ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* fast track a couple of common cases */ 5898ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 4 && aligned32(a)) { 5899ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply32_f__msmcread( thr, a ); 5900ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5901ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5902ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 8 && aligned64(a)) { 5903ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply64_f__msmcread( thr, a ); 5904ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5905ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5906ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5907ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* be completely general (but as efficient as possible) */ 5908ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5909ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5910ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!aligned16(a) && len >= 1) { 5911ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply08_f__msmcread( thr, a ); 5912ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 1; 5913ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 1; 5914ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned16(a)); 5915ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5916ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5917ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5918ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!aligned32(a) && len >= 2) { 5919ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply16_f__msmcread( thr, a ); 5920ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 2; 5921ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 2; 5922ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned32(a)); 5923ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5924ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5925ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5926ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!aligned64(a) && len >= 4) { 5927ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply32_f__msmcread( thr, a ); 5928ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 4; 5929ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 4; 5930ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned64(a)); 5931ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5932ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5933ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5934ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 8) { 5935ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned64(a)); 5936ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (len >= 8) { 5937ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply64_f__msmcread( thr, a ); 5938ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 8; 5939ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 8; 5940ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5941ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned64(a)); 5942ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5943ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5944ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5945ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 4) 5946ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned32(a)); 5947ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 4) { 5948ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply32_f__msmcread( thr, a ); 5949ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 4; 5950ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 4; 5951ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5952ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5953ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5954ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 2) 5955ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned16(a)); 5956ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 2) { 5957ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply16_f__msmcread( thr, a ); 5958ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 2; 5959ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 2; 5960ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5961ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5962ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5963ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 1) { 5964ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply08_f__msmcread( thr, a ); 5965ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //a += 1; 5966ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 1; 5967ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5968ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(len == 0); 5969ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5970ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5971ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid libhb_Thr_resumes ( Thr* thr ) 5972ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 5973ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)("resume %p\n", thr); 5974ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(thr); 5975b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(!thr->llexit_done); 5976ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Filter__clear(thr->filter, "libhb_Thr_resumes"); 5977ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* A kludge, but .. if this thread doesn't have any marker stacks 5978ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown at all, get one right now. This is easier than figuring out 5979ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown exactly when at thread startup we can and can't take a stack 5980ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown snapshot. */ 5981ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (HG_(clo_history_level) == 1) { 5982ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(thr->local_Kws_n_stacks); 5983ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (VG_(sizeXA)( thr->local_Kws_n_stacks ) == 0) 5984ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown note_local_Kw_n_stack_for(thr); 5985ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5986ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5987ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5988ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5989ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 5990ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 5991ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Synchronisation objects // 5992ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 5993ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 5994ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5995b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* A double linked list of all the SO's. */ 5996b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy StepanovSO* admin_SO = NULL; 5997ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5998b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic SO* SO__Alloc ( void ) 5999b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 6000ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SO* so = HG_(zalloc)( "libhb.SO__Alloc.1", sizeof(SO) ); 6001ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown so->viR = VtsID_INVALID; 6002ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown so->viW = VtsID_INVALID; 6003ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown so->magic = SO_MAGIC; 6004b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Add to double linked list */ 6005b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (admin_SO) { 6006b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(admin_SO->admin_prev == NULL); 6007b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov admin_SO->admin_prev = so; 6008b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov so->admin_next = admin_SO; 6009b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } else { 6010b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov so->admin_next = NULL; 6011b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 6012b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov so->admin_prev = NULL; 6013b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov admin_SO = so; 6014b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* */ 6015ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return so; 6016ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6017b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 6018b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void SO__Dealloc ( SO* so ) 6019b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 6020ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so); 6021ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so->magic == SO_MAGIC); 6022ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (so->viR == VtsID_INVALID) { 6023ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so->viW == VtsID_INVALID); 6024ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 6025ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so->viW != VtsID_INVALID); 6026ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcdec(so->viR); 6027ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcdec(so->viW); 6028ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6029ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown so->magic = 0; 6030b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Del from double linked list */ 6031b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (so->admin_prev) 6032b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov so->admin_prev->admin_next = so->admin_next; 6033b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (so->admin_next) 6034b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov so->admin_next->admin_prev = so->admin_prev; 6035b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (so == admin_SO) 6036b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov admin_SO = so->admin_next; 6037b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* */ 6038ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free)( so ); 6039ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6040ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6041ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6042ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 6043ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 6044ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Top Level API // 6045ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 6046ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 6047ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6048436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovstatic void show_thread_state ( const HChar* str, Thr* t ) 6049ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6050ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (1) return; 6051ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (t->viR == t->viW) { 6052ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("thr \"%s\" %p has vi* %u==", str, t, t->viR ); 6053ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__pp( t->viR ); 6054ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","\n"); 6055ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 6056ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("thr \"%s\" %p has viR %u==", str, t, t->viR ); 6057ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__pp( t->viR ); 6058ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" viW %u==", t->viW); 6059ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__pp( t->viW ); 6060ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","\n"); 6061ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6062ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6063ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6064ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6065ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownThr* libhb_init ( 6066ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown void (*get_stacktrace)( Thr*, Addr*, UWord ), 6067ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ExeContext* (*get_EC)( Thr* ) 6068ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ) 6069ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6070ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Thr* thr; 6071ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID vi; 6072b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 6073b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // We will have to have to store a large number of these, 6074b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // so make sure they're the size we expect them to be. 6075b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(sizeof(ScalarTS) == 8); 6076b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 6077b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* because first 1024 unusable */ 6078b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(SCALARTS_N_THRBITS >= 11); 6079b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* so as to fit in a UInt w/ 3 bits to spare (see defn of 6080b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Thr_n_RCEC). */ 6081b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(SCALARTS_N_THRBITS <= 29); 6082b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 6083b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Need to be sure that Thr_n_RCEC is 2 words (64-bit) or 3 words 6084b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov (32-bit). It's not correctness-critical, but there are a lot of 6085b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov them, so it's important from a space viewpoint. Unfortunately 6086b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov we simply can't pack it into 2 words on a 32-bit target. */ 6087b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (sizeof(UWord) == 8) { 6088b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(sizeof(Thr_n_RCEC) == 16); 6089b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } else { 6090b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(sizeof(Thr_n_RCEC) == 12); 6091b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 6092b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 6093b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Word sets really are 32 bits. Even on a 64 bit target. */ 6094b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(sizeof(WordSetID) == 4); 6095b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(sizeof(WordSet) == sizeof(WordSetID)); 6096b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 6097ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(get_stacktrace); 6098ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(get_EC); 6099ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown main_get_stacktrace = get_stacktrace; 6100ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown main_get_EC = get_EC; 6101ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6102ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // No need to initialise hg_wordfm. 6103ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // No need to initialise hg_wordset. 6104ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6105b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Allocated once and never deallocated. Used as a temporary in 6106b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS singleton, tick and join operations. */ 6107b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov temp_max_sized_VTS = VTS__new( "libhb.libhb_init.1", ThrID_MAX_VALID ); 6108b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov temp_max_sized_VTS->id = VtsID_INVALID; 6109b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov verydead_thread_table_init(); 6110ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts_set_init(); 6111ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts_tab_init(); 6112ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown event_map_init(); 6113ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__invalidate_caches(); 6114ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6115ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // initialise shadow memory 6116ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_init( SVal__rcinc, SVal__rcdec ); 6117ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6118ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr = Thr__new(); 6119ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vi = VtsID__mk_Singleton( thr, 1 ); 6120ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr->viR = vi; 6121ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr->viW = vi; 6122ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(thr->viR); 6123ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(thr->viW); 6124ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6125ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown show_thread_state(" root", thr); 6126ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return thr; 6127ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6128ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6129ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6130ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownThr* libhb_create ( Thr* parent ) 6131ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6132ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* The child's VTSs are copies of the parent's VTSs, but ticked at 6133ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown the child's index. Since the child's index is guaranteed 6134ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown unique, it has never been seen before, so the implicit value 6135ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown before the tick is zero and after that is one. */ 6136ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Thr* child = Thr__new(); 6137ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6138ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown child->viR = VtsID__tick( parent->viR, child ); 6139ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown child->viW = VtsID__tick( parent->viW, child ); 6140ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Filter__clear(child->filter, "libhb_create(child)"); 6141ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(child->viR); 6142ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(child->viW); 6143ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* We need to do note_local_Kw_n_stack_for( child ), but it's too 6144ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown early for that - it may not have a valid TId yet. So, let 6145ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown libhb_Thr_resumes pick it up the first time the thread runs. */ 6146ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6147ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(VtsID__indexAt( child->viR, child ) == 1); 6148ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(VtsID__indexAt( child->viW, child ) == 1); 6149ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6150ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* and the parent has to move along too */ 6151ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcdec(parent->viR); 6152ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcdec(parent->viW); 6153ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown parent->viR = VtsID__tick( parent->viR, parent ); 6154ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown parent->viW = VtsID__tick( parent->viW, parent ); 6155ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Filter__clear(parent->filter, "libhb_create(parent)"); 6156ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(parent->viR); 6157ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(parent->viW); 6158ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown note_local_Kw_n_stack_for( parent ); 6159ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6160ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown show_thread_state(" child", child); 6161ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown show_thread_state("parent", parent); 6162ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6163ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return child; 6164ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6165ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6166ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Shut down the library, and print stats (in fact that's _all_ 6167ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown this is for. */ 6168ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid libhb_shutdown ( Bool show_stats ) 6169ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6170ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (show_stats) { 6171ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","<<< BEGIN libhb stats >>>\n"); 6172ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" secmaps: %'10lu allocd (%'12lu g-a-range)\n", 6173ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmaps_allocd, 6174ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmap_ga_space_covered); 6175ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" linesZ: %'10lu allocd (%'12lu bytes occupied)\n", 6176ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmap_linesZ_allocd, 6177ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmap_linesZ_bytes); 6178ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" linesF: %'10lu allocd (%'12lu bytes occupied)\n", 6179ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmap_linesF_allocd, 6180ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmap_linesF_bytes); 6181ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" secmaps: %'10lu iterator steppings\n", 6182ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmap_iterator_steppings); 6183ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" secmaps: %'10lu searches (%'12lu slow)\n", 6184ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmaps_search, stats__secmaps_search_slow); 6185ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6186ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","\n"); 6187ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" cache: %'lu totrefs (%'lu misses)\n", 6188ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_totrefs, stats__cache_totmisses ); 6189ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" cache: %'14lu Z-fetch, %'14lu F-fetch\n", 6190ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_Z_fetches, stats__cache_F_fetches ); 6191ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" cache: %'14lu Z-wback, %'14lu F-wback\n", 6192ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_Z_wbacks, stats__cache_F_wbacks ); 6193ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" cache: %'14lu invals, %'14lu flushes\n", 6194ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_invals, stats__cache_flushes ); 6195ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" cache: %'14llu arange_New %'14llu direct-to-Zreps\n", 6196ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_make_New_arange, 6197ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_make_New_inZrep); 6198ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6199ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","\n"); 6200ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" cline: %'10lu normalises\n", 6201ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_normalises ); 6202ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" cline: c rds 8/4/2/1: %'13lu %'13lu %'13lu %'13lu\n", 6203ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cread64s, 6204ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cread32s, 6205ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cread16s, 6206ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cread08s ); 6207ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" cline: c wrs 8/4/2/1: %'13lu %'13lu %'13lu %'13lu\n", 6208ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cwrite64s, 6209ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cwrite32s, 6210ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cwrite16s, 6211ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cwrite08s ); 6212ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" cline: s wrs 8/4/2/1: %'13lu %'13lu %'13lu %'13lu\n", 6213ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_swrite64s, 6214ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_swrite32s, 6215ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_swrite16s, 6216ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_swrite08s ); 6217ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" cline: s rd1s %'lu, s copy1s %'lu\n", 6218ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_sread08s, stats__cline_scopy08s ); 6219ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" cline: splits: 8to4 %'12lu 4to2 %'12lu 2to1 %'12lu\n", 6220ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_64to32splits, 6221ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_32to16splits, 6222ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_16to8splits ); 6223ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" cline: pulldowns: 8to4 %'12lu 4to2 %'12lu 2to1 %'12lu\n", 6224ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_64to32pulldown, 6225ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_32to16pulldown, 6226ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_16to8pulldown ); 6227ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) 6228ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" cline: sizeof(CacheLineZ) %ld, covers %ld bytes of arange\n", 6229ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Word)sizeof(LineZ), (Word)N_LINE_ARANGE); 6230ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6231ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","\n"); 6232ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6233ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" libhb: %'13llu msmcread (%'llu dragovers)\n", 6234ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__msmcread, stats__msmcread_change); 6235ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" libhb: %'13llu msmcwrite (%'llu dragovers)\n", 6236ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__msmcwrite, stats__msmcwrite_change); 6237ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" libhb: %'13llu cmpLEQ queries (%'llu misses)\n", 6238ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cmpLEQ_queries, stats__cmpLEQ_misses); 6239ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" libhb: %'13llu join2 queries (%'llu misses)\n", 6240ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__join2_queries, stats__join2_misses); 6241ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6242ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","\n"); 6243ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( " libhb: VTSops: tick %'lu, join %'lu, cmpLEQ %'lu\n", 6244ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__vts__tick, stats__vts__join, stats__vts__cmpLEQ ); 6245ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( " libhb: VTSops: cmp_structural %'lu (%'lu slow)\n", 6246ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__vts__cmp_structural, stats__vts__cmp_structural_slow ); 6247b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(printf)( " libhb: VTSset: find__or__clone_and_add %'lu (%'lu allocd)\n", 6248b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov stats__vts_set__focaa, stats__vts_set__focaa_a ); 6249ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( " libhb: VTSops: indexAt_SLOW %'lu\n", 6250ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__vts__indexat_slow ); 6251ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6252ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","\n"); 6253ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( 6254ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown " libhb: %ld entries in vts_table (approximately %lu bytes)\n", 6255ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(sizeXA)( vts_tab ), VG_(sizeXA)( vts_tab ) * sizeof(VtsTE) 6256ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ); 6257ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( " libhb: %lu entries in vts_set\n", 6258ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(sizeFM)( vts_set ) ); 6259ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6260ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","\n"); 6261ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( " libhb: ctxt__rcdec: 1=%lu(%lu eq), 2=%lu, 3=%lu\n", 6262ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_rcdec1, stats__ctxt_rcdec1_eq, 6263ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_rcdec2, 6264ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_rcdec3 ); 6265ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( " libhb: ctxt__rcdec: calls %lu, discards %lu\n", 6266ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_rcdec_calls, stats__ctxt_rcdec_discards); 6267ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( " libhb: contextTab: %lu slots, %lu max ents\n", 6268ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (UWord)N_RCEC_TAB, 6269ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_tab_curr ); 6270ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( " libhb: contextTab: %lu queries, %lu cmps\n", 6271ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_tab_qs, 6272ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_tab_cmps ); 6273ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#if 0 6274ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(AvlNode) = %lu\n", sizeof(AvlNode)); 6275ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(WordBag) = %lu\n", sizeof(WordBag)); 6276ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(MaybeWord) = %lu\n", sizeof(MaybeWord)); 6277ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(CacheLine) = %lu\n", sizeof(CacheLine)); 6278ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(LineZ) = %lu\n", sizeof(LineZ)); 6279ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(LineF) = %lu\n", sizeof(LineF)); 6280ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(SecMap) = %lu\n", sizeof(SecMap)); 6281ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(Cache) = %lu\n", sizeof(Cache)); 6282ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(SMCacheEnt) = %lu\n", sizeof(SMCacheEnt)); 6283ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(CountedSVal) = %lu\n", sizeof(CountedSVal)); 6284ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(VTS) = %lu\n", sizeof(VTS)); 6285ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(ScalarTS) = %lu\n", sizeof(ScalarTS)); 6286ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(VtsTE) = %lu\n", sizeof(VtsTE)); 6287ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(MSMInfo) = %lu\n", sizeof(MSMInfo)); 6288ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6289ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(struct _XArray) = %lu\n", sizeof(struct _XArray)); 6290ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(struct _WordFM) = %lu\n", sizeof(struct _WordFM)); 6291ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(struct _Thr) = %lu\n", sizeof(struct _Thr)); 6292ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(struct _SO) = %lu\n", sizeof(struct _SO)); 6293ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#endif 6294ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6295ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","<<< END libhb stats >>>\n"); 6296ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","\n"); 6297ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6298ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6299ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6300ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6301b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Receive notification that a thread has low level exited. The 6302b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov significance here is that we do not expect to see any more memory 6303b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov references from it. */ 6304ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid libhb_async_exit ( Thr* thr ) 6305ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6306ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(thr); 6307b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(!thr->llexit_done); 6308b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thr->llexit_done = True; 6309ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6310ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* free up Filter and local_Kws_n_stacks (well, actually not the 6311ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown latter ..) */ 6312ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(thr->filter); 6313ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free)(thr->filter); 6314ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr->filter = NULL; 6315ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6316b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Tell the VTS mechanism this thread has exited, so it can 6317b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov participate in VTS pruning. Note this can only happen if the 6318b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thread has both ll_exited and has been joined with. */ 6319b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (thr->joinedwith_done) 6320b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS__declare_thread_very_dead(thr); 6321b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 6322ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Another space-accuracy tradeoff. Do we want to be able to show 6323ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown H1 history for conflicts in threads which have since exited? If 6324ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown yes, then we better not free up thr->local_Kws_n_stacks. The 6325ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown downside is a potential per-thread leak of up to 6326ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown N_KWs_N_STACKs_PER_THREAD * sizeof(ULong_n_EC) * whatever the 6327ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown XArray average overcommit factor is (1.5 I'd guess). */ 6328ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // hence: 6329ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // VG_(deleteXA)(thr->local_Kws_n_stacks); 6330ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // thr->local_Kws_n_stacks = NULL; 6331ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6332ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6333b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Receive notification that a thread has been joined with. The 6334b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov significance here is that we do not expect to see any further 6335b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov references to its vector clocks (Thr::viR and Thr::viW). */ 6336b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovvoid libhb_joinedwith_done ( Thr* thr ) 6337b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 6338b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(thr); 6339b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Caller must ensure that this is only ever called once per Thr. */ 6340b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(!thr->joinedwith_done); 6341b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thr->joinedwith_done = True; 6342b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (thr->llexit_done) 6343b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS__declare_thread_very_dead(thr); 6344b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 6345b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 6346b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 6347ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Both Segs and SOs point to VTSs. However, there is no sharing, so 6348ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a Seg that points at a VTS is its one-and-only owner, and ditto for 6349ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a SO that points at a VTS. */ 6350ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6351ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownSO* libhb_so_alloc ( void ) 6352ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6353ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return SO__Alloc(); 6354ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6355ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6356ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid libhb_so_dealloc ( SO* so ) 6357ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6358ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so); 6359ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so->magic == SO_MAGIC); 6360ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SO__Dealloc(so); 6361ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6362ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6363ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* See comments in libhb.h for details on the meaning of 6364ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown strong vs weak sends and strong vs weak receives. */ 6365ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid libhb_so_send ( Thr* thr, SO* so, Bool strong_send ) 6366ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6367ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Copy the VTSs from 'thr' into the sync object, and then move 6368ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown the thread along one step. */ 6369ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6370ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so); 6371ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so->magic == SO_MAGIC); 6372ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6373ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* stay sane .. a thread's read-clock must always lead or be the 6374ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown same as its write-clock */ 6375ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown { Bool leq = VtsID__cmpLEQ(thr->viW, thr->viR); 6376ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(leq); 6377ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6378ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6379ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* since we're overwriting the VtsIDs in the SO, we need to drop 6380ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown any references made by the previous contents thereof */ 6381ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (so->viR == VtsID_INVALID) { 6382ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so->viW == VtsID_INVALID); 6383ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown so->viR = thr->viR; 6384ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown so->viW = thr->viW; 6385ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(so->viR); 6386ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(so->viW); 6387ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 6388ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* In a strong send, we dump any previous VC in the SO and 6389ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown install the sending thread's VC instead. For a weak send we 6390ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown must join2 with what's already there. */ 6391ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so->viW != VtsID_INVALID); 6392ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcdec(so->viR); 6393ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcdec(so->viW); 6394ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown so->viR = strong_send ? thr->viR : VtsID__join2( so->viR, thr->viR ); 6395ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown so->viW = strong_send ? thr->viW : VtsID__join2( so->viW, thr->viW ); 6396ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(so->viR); 6397ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(so->viW); 6398ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6399ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6400ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* move both parent clocks along */ 6401ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcdec(thr->viR); 6402ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcdec(thr->viW); 6403ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr->viR = VtsID__tick( thr->viR, thr ); 6404ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr->viW = VtsID__tick( thr->viW, thr ); 6405b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (!thr->llexit_done) { 6406ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Filter__clear(thr->filter, "libhb_so_send"); 6407ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown note_local_Kw_n_stack_for(thr); 6408ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6409ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(thr->viR); 6410ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(thr->viW); 6411ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6412ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (strong_send) 6413ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown show_thread_state("s-send", thr); 6414ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown else 6415ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown show_thread_state("w-send", thr); 6416ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6417ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6418ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid libhb_so_recv ( Thr* thr, SO* so, Bool strong_recv ) 6419ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6420ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so); 6421ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so->magic == SO_MAGIC); 6422ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6423ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (so->viR != VtsID_INVALID) { 6424ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so->viW != VtsID_INVALID); 6425ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6426ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Weak receive (basically, an R-acquisition of a R-W lock). 6427ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown This advances the read-clock of the receiver, but not the 6428ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write-clock. */ 6429ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcdec(thr->viR); 6430ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr->viR = VtsID__join2( thr->viR, so->viR ); 6431ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(thr->viR); 6432ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6433ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* At one point (r10589) it seemed safest to tick the clocks for 6434ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown the receiving thread after the join. But on reflection, I 6435ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown wonder if that might cause it to 'overtake' constraints, 6436ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown which could lead to missing races. So, back out that part of 6437ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown r10589. */ 6438ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //VtsID__rcdec(thr->viR); 6439ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //thr->viR = VtsID__tick( thr->viR, thr ); 6440ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //VtsID__rcinc(thr->viR); 6441ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6442ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* For a strong receive, we also advance the receiver's write 6443ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown clock, which means the receive as a whole is essentially 6444ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown equivalent to a W-acquisition of a R-W lock. */ 6445ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (strong_recv) { 6446ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcdec(thr->viW); 6447ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr->viW = VtsID__join2( thr->viW, so->viW ); 6448ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(thr->viW); 6449ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6450ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* See comment just above, re r10589. */ 6451ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //VtsID__rcdec(thr->viW); 6452ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //thr->viW = VtsID__tick( thr->viW, thr ); 6453ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //VtsID__rcinc(thr->viW); 6454ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6455ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6456ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (thr->filter) 6457ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Filter__clear(thr->filter, "libhb_so_recv"); 6458ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown note_local_Kw_n_stack_for(thr); 6459ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6460ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (strong_recv) 6461ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown show_thread_state("s-recv", thr); 6462ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown else 6463ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown show_thread_state("w-recv", thr); 6464ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6465ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 6466ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so->viW == VtsID_INVALID); 6467ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Deal with degenerate case: 'so' has no vts, so there has been 6468ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown no message posted to it. Just ignore this case. */ 6469ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown show_thread_state("d-recv", thr); 6470ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6471ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6472ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6473ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownBool libhb_so_everSent ( SO* so ) 6474ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6475ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (so->viR == VtsID_INVALID) { 6476ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so->viW == VtsID_INVALID); 6477ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 6478ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 6479ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so->viW != VtsID_INVALID); 6480ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return True; 6481ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6482ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6483ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6484ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define XXX1 0 // 0x67a106c 6485ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define XXX2 0 6486ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6487ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool TRACEME(Addr a, SizeT szB) { 6488ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (XXX1 && a <= XXX1 && XXX1 <= a+szB) return True; 6489ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (XXX2 && a <= XXX2 && XXX2 <= a+szB) return True; 6490ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 6491ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6492436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovstatic void trace ( Thr* thr, Addr a, SizeT szB, const HChar* s ) 6493436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov{ 6494ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal sv = zsm_sread08(a); 6495ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("thr %p (%#lx,%lu) %s: 0x%016llx ", thr,a,szB,s,sv); 6496ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown show_thread_state("", thr); 6497ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","\n"); 6498ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6499ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6500ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid libhb_srange_new ( Thr* thr, Addr a, SizeT szB ) 6501ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6502ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal sv = SVal__mkC(thr->viW, thr->viW); 6503ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_SVal_C(sv)); 6504ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0 && TRACEME(a,szB)) trace(thr,a,szB,"nw-before"); 6505ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sset_range( a, szB, sv ); 6506ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Filter__clear_range( thr->filter, a, szB ); 6507ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0 && TRACEME(a,szB)) trace(thr,a,szB,"nw-after "); 6508ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6509ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6510b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovvoid libhb_srange_noaccess_NoFX ( Thr* thr, Addr a, SizeT szB ) 6511ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6512ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* do nothing */ 6513ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6514ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6515b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovvoid libhb_srange_noaccess_AHAE ( Thr* thr, Addr a, SizeT szB ) 6516b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 6517b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* This really does put the requested range in NoAccess. It's 6518b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov expensive though. */ 6519b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov SVal sv = SVal_NOACCESS; 6520b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(is_sane_SVal_C(sv)); 6521b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov zsm_sset_range( a, szB, sv ); 6522b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Filter__clear_range( thr->filter, a, szB ); 6523b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 6524b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 6525ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid libhb_srange_untrack ( Thr* thr, Addr a, SizeT szB ) 6526ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6527ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal sv = SVal_NOACCESS; 6528ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_SVal_C(sv)); 6529ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0 && TRACEME(a,szB)) trace(thr,a,szB,"untrack-before"); 6530ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sset_range( a, szB, sv ); 6531ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Filter__clear_range( thr->filter, a, szB ); 6532ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0 && TRACEME(a,szB)) trace(thr,a,szB,"untrack-after "); 6533ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6534ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6535b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy StepanovThread* libhb_get_Thr_hgthread ( Thr* thr ) { 6536ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(thr); 6537b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return thr->hgthread; 6538ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6539ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6540b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovvoid libhb_set_Thr_hgthread ( Thr* thr, Thread* hgthread ) { 6541ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(thr); 6542b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thr->hgthread = hgthread; 6543ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6544ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6545ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid libhb_copy_shadow_state ( Thr* thr, Addr src, Addr dst, SizeT len ) 6546ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6547ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_scopy_range(src, dst, len); 6548ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Filter__clear_range( thr->filter, dst, len ); 6549ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6550ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6551ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid libhb_maybe_GC ( void ) 6552ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6553ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown event_map_maybe_GC(); 6554ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* If there are still freelist entries available, no need for a 6555ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown GC. */ 6556ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (vts_tab_freelist != VtsID_INVALID) 6557ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 6558ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* So all the table entries are full, and we're having to expand 6559ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown the table. But did we hit the threshhold point yet? */ 6560ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (VG_(sizeXA)( vts_tab ) < vts_next_GC_at) 6561ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 6562ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts_tab__do_GC( False/*don't show stats*/ ); 6563ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6564ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6565ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6566ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 6567ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 6568ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 6569ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// SECTION END main library // 6570ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 6571ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 6572ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 6573ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6574ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--------------------------------------------------------------------*/ 6575ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--- end libhb_main.c ---*/ 6576ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--------------------------------------------------------------------*/ 6577