1ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--------------------------------------------------------------------*/ 3ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--- LibHB: a library for implementing and checking ---*/ 4ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--- the happens-before relationship in concurrent programs. ---*/ 5ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--- libhb_main.c ---*/ 6ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--------------------------------------------------------------------*/ 7ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 8ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* 9ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown This file is part of LibHB, a library for implementing and checking 10ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown the happens-before relationship in concurrent programs. 11ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 12b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Copyright (C) 2008-2011 OpenWorks Ltd 13ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown info@open-works.co.uk 14ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 15ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown This program is free software; you can redistribute it and/or 16ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown modify it under the terms of the GNU General Public License as 17ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown published by the Free Software Foundation; either version 2 of the 18ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown License, or (at your option) any later version. 19ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 20ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown This program is distributed in the hope that it will be useful, but 21ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown WITHOUT ANY WARRANTY; without even the implied warranty of 22ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 23ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown General Public License for more details. 24ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 25ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown You should have received a copy of the GNU General Public License 26ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown along with this program; if not, write to the Free Software 27ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 28ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 02111-1307, USA. 29ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 30ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown The GNU General Public License is contained in the file COPYING. 31ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 32ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 33ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_basics.h" 34ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_libcassert.h" 35ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_libcbase.h" 36ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_libcprint.h" 37ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_mallocfree.h" 38ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_wordfm.h" 39ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_sparsewa.h" 40ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_xarray.h" 41ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_oset.h" 42ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_threadstate.h" 43ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_aspacemgr.h" 44ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_execontext.h" 45ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_errormgr.h" 46ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_tool_options.h" // VG_(clo_stats) 47ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "hg_basics.h" 48ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "hg_wordset.h" 49ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "hg_lock_n_thread.h" 50ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "hg_errors.h" 51ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 52ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "libhb.h" 53ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 54ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 55ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 56ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 57ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 58ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Debugging #defines // 59ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 60ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 61ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 62ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 63ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Check the sanity of shadow values in the core memory state 64ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown machine. Change #if 0 to #if 1 to enable this. */ 65ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#if 0 66ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# define CHECK_MSM 1 67ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#else 68ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# define CHECK_MSM 0 69ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#endif 70ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 71ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 72ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Check sanity (reference counts, etc) in the conflicting access 73ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown machinery. Change #if 0 to #if 1 to enable this. */ 74ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#if 0 75ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# define CHECK_CEM 1 76ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#else 77ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# define CHECK_CEM 0 78ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#endif 79ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 80ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 81ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Check sanity in the compressed shadow memory machinery, 82ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown particularly in its caching innards. Unfortunately there's no 83ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown almost-zero-cost way to make them selectable at run time. Hence 84ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown set the #if 0 to #if 1 and rebuild if you want them. */ 85ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#if 0 86ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# define CHECK_ZSM 1 /* do sanity-check CacheLine stuff */ 87ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# define inline __attribute__((noinline)) 88ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* probably want to ditch -fomit-frame-pointer too */ 89ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#else 90ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# define CHECK_ZSM 0 /* don't sanity-check CacheLine stuff */ 91ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#endif 92ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 93ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 94ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 95ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 96ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 97b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// data decls: VtsID // 98ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 99ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 100ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 101ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 102b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* VtsIDs: Unique small-integer IDs for VTSs. VtsIDs can't exceed 30 103b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov bits, since they have to be packed into the lowest 30 bits of an 104b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov SVal. */ 105b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovtypedef UInt VtsID; 106b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define VtsID_INVALID 0xFFFFFFFF 107ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 108ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 109ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 110ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 111ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 112ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 113b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// data decls: SVal // 114ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 115ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 116ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 117ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 118ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef ULong SVal; 119ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 120ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* This value has special significance to the implementation, and callers 121ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown may not store it in the shadow memory. */ 122ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define SVal_INVALID (3ULL << 62) 123ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 124ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* This is the default value for shadow memory. Initially the shadow 125ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown memory contains no accessible areas and so all reads produce this 126ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown value. TODO: make this caller-defineable. */ 127ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define SVal_NOACCESS (2ULL << 62) 128ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 129b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 130b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 131b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 132b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 133b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// // 134b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// data decls: ScalarTS // 135b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// // 136b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 137b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 138b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 139b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Scalar Timestamp. We have to store a lot of these, so there is 140b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov some effort to make them as small as possible. Logically they are 141b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a pair, (Thr*, ULong), but that takes 16 bytes on a 64-bit target. 142b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov We pack it into 64 bits by representing the Thr* using a ThrID, a 143b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov small integer (18 bits), and a 46 bit integer for the timestamp 144b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov number. The 46/18 split is arbitary, but has the effect that 145b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Helgrind can only handle programs that create 2^18 or fewer threads 146b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov over their entire lifetime, and have no more than 2^46 timestamp 147b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ticks (synchronisation operations on the same thread). 148b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 149b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov This doesn't seem like much of a limitation. 2^46 ticks is 150b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 7.06e+13, and if each tick (optimistically) takes the machine 1000 151b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cycles to process, then the minimum time to process that many ticks 152b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov at a clock rate of 5 GHz is 162.9 days. And that's doing nothing 153b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov but VTS ticks, which isn't realistic. 154b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 155b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov NB1: SCALARTS_N_THRBITS must be 29 or lower. The obvious limit is 156b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 32 since a ThrID is a UInt. 29 comes from the fact that 157b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 'Thr_n_RCEC', which records information about old accesses, packs 158b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov not only a ThrID but also 2+1 other bits (access size and 159b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov writeness) in a UInt, hence limiting size to 32-(2+1) == 29. 160b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 161b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov NB2: thrid values are issued upwards from 1024, and values less 162b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov than that aren't valid. This isn't per se necessary (any order 163b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov will do, so long as they are unique), but it does help ensure they 164b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov are less likely to get confused with the various other kinds of 165b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov small-integer thread ids drifting around (eg, TId). See also NB5. 166b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 167b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov NB3: this probably also relies on the fact that Thr's are never 168b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov deallocated -- they exist forever. Hence the 1-1 mapping from 169b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Thr's to thrid values (set up in Thr__new) persists forever. 170b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 171b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov NB4: temp_max_sized_VTS is allocated at startup and never freed. 172b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov It is a maximum sized VTS, so has (1 << SCALARTS_N_TYMBITS) 173b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTSs. So we can't make SCALARTS_N_THRBITS too large without 174b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov making the memory use for this go sky-high. With 175b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov SCALARTS_N_THRBITS at 18, it occupies 2MB of memory, which seems 176b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov like an OK tradeoff. If more than 256k threads need to be 177b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov supported, we could change SCALARTS_N_THRBITS to 20, which would 178b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov facilitate supporting 1 million threads at the cost of 8MB storage 179b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for temp_max_sized_VTS. 180b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 181b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov NB5: the conflicting-map mechanism (Thr_n_RCEC, specifically) uses 182b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID == 0 to denote an empty Thr_n_RCEC record. So ThrID == 0 183b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov must never be a valid ThrID. Given NB2 that's OK. 184b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov*/ 185b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define SCALARTS_N_THRBITS 18 /* valid range: 11 to 29 inclusive */ 186b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 187b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define SCALARTS_N_TYMBITS (64 - SCALARTS_N_THRBITS) 188b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovtypedef 189b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov struct { 190b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID thrid : SCALARTS_N_THRBITS; 191b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ULong tym : SCALARTS_N_TYMBITS; 192b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 193b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS; 194b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 195b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define ThrID_MAX_VALID ((1 << SCALARTS_N_THRBITS) - 1) 196b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 197b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 198b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 199b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 200b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 201b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// // 202b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// data decls: Filter // 203b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// // 204b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 205b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 206b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 207b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// baseline: 5, 9 208b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define FI_LINE_SZB_LOG2 5 209b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define FI_NUM_LINES_LOG2 10 210b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 211b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define FI_LINE_SZB (1 << FI_LINE_SZB_LOG2) 212b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define FI_NUM_LINES (1 << FI_NUM_LINES_LOG2) 213b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 214b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define FI_TAG_MASK (~(Addr)(FI_LINE_SZB - 1)) 215b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define FI_GET_TAG(_a) ((_a) & FI_TAG_MASK) 216b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 217b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define FI_GET_LINENO(_a) ( ((_a) >> FI_LINE_SZB_LOG2) \ 218b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov & (Addr)(FI_NUM_LINES-1) ) 219b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 220b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 221b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* In the lines, each 8 bytes are treated individually, and are mapped 222b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov to a UShort. Regardless of endianness of the underlying machine, 223b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov bits 1 and 0 pertain to the lowest address and bits 15 and 14 to 224b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov the highest address. 225b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 226b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Of each bit pair, the higher numbered bit is set if a R has been 227b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov seen, so the actual layout is: 228b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 229b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 15 14 ... 01 00 230b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 231b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov R W for addr+7 ... R W for addr+0 232b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 233b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov So a mask for the R-bits is 0xAAAA and for the W bits is 0x5555. 234b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov*/ 235b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 236b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* tags are separated from lines. tags are Addrs and are 237b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov the base address of the line. */ 238b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovtypedef 239b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov struct { 240b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UShort u16s[FI_LINE_SZB / 8]; /* each UShort covers 8 bytes */ 241b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 242b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov FiLine; 243b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 244b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovtypedef 245b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov struct { 246b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Addr tags[FI_NUM_LINES]; 247b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov FiLine lines[FI_NUM_LINES]; 248b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 249b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Filter; 250b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 251b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 252b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 253b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 254b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 255b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// // 256b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// data decls: Thr, ULong_n_EC // 257b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// // 258b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 259b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 260b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 261b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// Records stacks for H1 history mechanism (DRD-style) 262b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovtypedef 263b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov struct { ULong ull; ExeContext* ec; } 264b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ULong_n_EC; 265b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 266b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 267b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* How many of the above records to collect for each thread? Older 268b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ones are dumped when we run out of space. 62.5k requires 1MB per 269b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thread, since each ULong_n_EC record is 16 bytes long. When more 270b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov than N_KWs_N_STACKs_PER_THREAD are present, the older half are 271b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov deleted to make space. Hence in the worst case we will be able to 272b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov produce a stack at least for the last N_KWs_N_STACKs_PER_THREAD / 2 273b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Kw transitions (segments in this thread). For the current setting 274b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov that gives a guaranteed stack for at least the last 31.25k 275b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov segments. */ 276b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define N_KWs_N_STACKs_PER_THREAD 62500 277b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 278b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 279b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstruct _Thr { 280b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Current VTSs for this thread. They change as we go along. viR 281b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov is the VTS to be used for reads, viW for writes. Usually they 282b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov are the same, but can differ when we deal with reader-writer 283b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov locks. It is always the case that 284b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID__cmpLEQ(viW,viR) == True 285b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov that is, viW must be the same, or lagging behind, viR. */ 286b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID viR; 287b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID viW; 288b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 289b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Is initially False, and is set to True after the thread really 290b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov has done a low-level exit. When True, we expect to never see 291b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov any more memory references done by this thread. */ 292b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Bool llexit_done; 293b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 294b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Is initially False, and is set to True after the thread has been 295b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov joined with (reaped by some other thread). After this point, we 296b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov do not expect to see any uses of .viR or .viW, so it is safe to 297b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov set them to VtsID_INVALID. */ 298b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Bool joinedwith_done; 299b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 300b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* A small integer giving a unique identity to this Thr. See 301b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov comments on the definition of ScalarTS for details. */ 302b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID thrid : SCALARTS_N_THRBITS; 303b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 304b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* A filter that removes references for which we believe that 305b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov msmcread/msmcwrite will not change the state, nor report a 306b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov race. */ 307b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Filter* filter; 308b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 309b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* A pointer back to the top level Thread structure. There is a 310b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1-1 mapping between Thread and Thr structures -- each Thr points 311b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov at its corresponding Thread, and vice versa. Really, Thr and 312b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Thread should be merged into a single structure. */ 313b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Thread* hgthread; 314b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 315b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* The ULongs (scalar Kws) in this accumulate in strictly 316b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov increasing order, without duplicates. This is important because 317b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov we need to be able to find a given scalar Kw in this array 318b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov later, by binary search. */ 319b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov XArray* /* ULong_n_EC */ local_Kws_n_stacks; 320b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov}; 321b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 322b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 323b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 324b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 325b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 326b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// // 327b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// data decls: SO // 328b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// // 329b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 330b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 331b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 332b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// (UInt) `echo "Synchronisation object" | md5sum` 333b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define SO_MAGIC 0x56b3c5b0U 334b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 335b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstruct _SO { 336b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov struct _SO* admin_prev; 337b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov struct _SO* admin_next; 338b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID viR; /* r-clock of sender */ 339b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID viW; /* w-clock of sender */ 340b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt magic; 341b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov}; 342b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 343b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 344b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 345b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 346b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 347b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// // 348b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// Forward declarations // 349b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// // 350b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 351b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 352b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 353b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* fwds for 354b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Globals needed by other parts of the library. These are set 355b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov once at startup and then never changed. */ 356b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void (*main_get_stacktrace)( Thr*, Addr*, UWord ) = NULL; 357b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic ExeContext* (*main_get_EC)( Thr* ) = NULL; 358b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 359b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* misc fn and data fwdses */ 360b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void VtsID__rcinc ( VtsID ii ); 361b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void VtsID__rcdec ( VtsID ii ); 362b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 363b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic inline Bool SVal__isC ( SVal s ); 364b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic inline VtsID SVal__unC_Rmin ( SVal s ); 365b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic inline VtsID SVal__unC_Wmin ( SVal s ); 366b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic inline SVal SVal__mkC ( VtsID rmini, VtsID wmini ); 367b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 368b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* A double linked list of all the SO's. */ 369b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy StepanovSO* admin_SO; 370b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 371b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 372b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 373b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 374b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 375b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// // 376b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// SECTION BEGIN compressed shadow memory // 377b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// // 378b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 379b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov///////////////////////////////////////////////////////////////// 380b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 381b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#ifndef __HB_ZSM_H 382b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define __HB_ZSM_H 383b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 384ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Initialise the library. Once initialised, it will (or may) call 385ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcinc and rcdec in response to all the calls below, in order to 386ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown allow the user to do reference counting on the SVals stored herein. 387ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown It is important to understand, however, that due to internal 388ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown caching, the reference counts are in general inaccurate, and can be 389ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown both above or below the true reference count for an item. In 390ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown particular, the library may indicate that the reference count for 391ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown an item is zero, when in fact it is not. 392ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 393ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown To make the reference counting exact and therefore non-pointless, 394ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown call zsm_flush_cache. Immediately after it returns, the reference 395ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown counts for all items, as deduced by the caller by observing calls 396ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown to rcinc and rcdec, will be correct, and so any items with a zero 397ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown reference count may be freed (or at least considered to be 398ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown unreferenced by this library). 399ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 400ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_init ( void(*rcinc)(SVal), void(*rcdec)(SVal) ); 401ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 402ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_sset_range ( Addr, SizeT, SVal ); 403ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_scopy_range ( Addr, Addr, SizeT ); 404ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_flush_cache ( void ); 405ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 406ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#endif /* ! __HB_ZSM_H */ 407ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 408ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 409ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Round a up to the next multiple of N. N must be a power of 2 */ 410ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define ROUNDUP(a, N) ((a + N - 1) & ~(N-1)) 411ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Round a down to the next multiple of N. N must be a power of 2 */ 412ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define ROUNDDN(a, N) ((a) & ~(N-1)) 413ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 414ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 415ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 416ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* ------ User-supplied RC functions ------ */ 417ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void(*rcinc)(SVal) = NULL; 418ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void(*rcdec)(SVal) = NULL; 419ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 420ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 421ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* ------ CacheLine ------ */ 422ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 423ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_LINE_BITS 6 /* must be >= 3 */ 424ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_LINE_ARANGE (1 << N_LINE_BITS) 425ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_LINE_TREES (N_LINE_ARANGE >> 3) 426ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 427ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef 428ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct { 429ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descrs[N_LINE_TREES]; 430ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal svals[N_LINE_ARANGE]; // == N_LINE_TREES * 8 431ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 432ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine; 433ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 434ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_16_0 (1<<0) 435ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_32_0 (1<<1) 436ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_16_1 (1<<2) 437ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_64 (1<<3) 438ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_16_2 (1<<4) 439ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_32_1 (1<<5) 440ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_16_3 (1<<6) 441ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_8_0 (1<<7) 442ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_8_1 (1<<8) 443ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_8_2 (1<<9) 444ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_8_3 (1<<10) 445ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_8_4 (1<<11) 446ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_8_5 (1<<12) 447ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_8_6 (1<<13) 448ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_8_7 (1<<14) 449ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define TREE_DESCR_DTY (1<<15) 450ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 451ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef 452ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct { 453ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal dict[4]; /* can represent up to 4 diff values in the line */ 454ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UChar ix2s[N_LINE_ARANGE/4]; /* array of N_LINE_ARANGE 2-bit 455ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown dict indexes */ 456ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* if dict[0] == SVal_INVALID then dict[1] is the index of the 457ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineF to use, and dict[2..] are also SVal_INVALID. */ 458ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 459ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineZ; /* compressed rep for a cache line */ 460ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 461ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef 462ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct { 463ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool inUse; 464ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal w64s[N_LINE_ARANGE]; 465ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 466ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineF; /* full rep for a cache line */ 467ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 468ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Shadow memory. 469ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Primary map is a WordFM Addr SecMap*. 470ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SecMaps cover some page-size-ish section of address space and hold 471ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a compressed representation. 472ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine-sized chunks of SecMaps are copied into a Cache, being 473ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown decompressed when moved into the cache and recompressed on the 474ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown way out. Because of this, the cache must operate as a writeback 475ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cache, not a writethrough one. 476ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 477ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Each SecMap must hold a power-of-2 number of CacheLines. Hence 478ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown N_SECMAP_BITS must >= N_LINE_BITS. 479ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 480ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_SECMAP_BITS 13 481ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_SECMAP_ARANGE (1 << N_SECMAP_BITS) 482ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 483ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// # CacheLines held by a SecMap 484ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_SECMAP_ZLINES (N_SECMAP_ARANGE / N_LINE_ARANGE) 485ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 486ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* The data in the SecMap is held in the array of LineZs. Each LineZ 487ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown either carries the required data directly, in a compressed 488ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown representation, or it holds (in .dict[0]) an index to the LineF in 489ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown .linesF that holds the full representation. 490ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 491ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Currently-unused LineF's have their .inUse bit set to zero. 492ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Since each in-use LineF is referred to be exactly one LineZ, 493ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown the number of .linesZ[] that refer to .linesF should equal 494ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown the number of .linesF[] that have .inUse == True. 495ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 496ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown RC obligations: the RCs presented to the user include exactly 497ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown the values in: 498ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown * direct Z reps, that is, ones for which .dict[0] != SVal_INVALID 499ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown * F reps that are in use (.inUse == True) 500ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 501ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Hence the following actions at the following transitions are required: 502ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 503ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown F rep: .inUse==True -> .inUse==False -- rcdec_LineF 504ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown F rep: .inUse==False -> .inUse==True -- rcinc_LineF 505ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Z rep: .dict[0] from other to SVal_INVALID -- rcdec_LineZ 506ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Z rep: .dict[0] from SVal_INVALID to other -- rcinc_LineZ 507ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 508ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef 509ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct { 510ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt magic; 511ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineZ linesZ[N_SECMAP_ZLINES]; 512ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineF* linesF; 513ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt linesF_size; 514ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 515ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SecMap; 516ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 517ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define SecMap_MAGIC 0x571e58cbU 518ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 519ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool is_sane_SecMap ( SecMap* sm ) { 520ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return sm != NULL && sm->magic == SecMap_MAGIC; 521ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 522ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 523ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* ------ Cache ------ */ 524ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 525ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_WAY_BITS 16 526ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_WAY_NENT (1 << N_WAY_BITS) 527ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 528ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Each tag is the address of the associated CacheLine, rounded down 529ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown to a CacheLine address boundary. A CacheLine size must be a power 530ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown of 2 and must be 8 or more. Hence an easy way to initialise the 531ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cache so it is empty is to set all the tag values to any value % 8 532ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown != 0, eg 1. This means all queries in the cache initially miss. 533ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown It does however require us to detect and not writeback, any line 534ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown with a bogus tag. */ 535ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef 536ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct { 537ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine lyns0[N_WAY_NENT]; 538ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr tags0[N_WAY_NENT]; 539ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 540ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Cache; 541ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 542ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool is_valid_scache_tag ( Addr tag ) { 543ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* a valid tag should be naturally aligned to the start of 544ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a CacheLine. */ 545ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0 == (tag & (N_LINE_ARANGE - 1)); 546ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 547ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 548ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 549ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* --------- Primary data structures --------- */ 550ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 551ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Shadow memory primary map */ 552ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic WordFM* map_shmem = NULL; /* WordFM Addr SecMap* */ 553ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Cache cache_shmem; 554ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 555ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 556ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__secmaps_search = 0; // # SM finds 557ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__secmaps_search_slow = 0; // # SM lookupFMs 558ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__secmaps_allocd = 0; // # SecMaps issued 559ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__secmap_ga_space_covered = 0; // # ga bytes covered 560ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__secmap_linesZ_allocd = 0; // # LineZ's issued 561ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__secmap_linesZ_bytes = 0; // .. using this much storage 562ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__secmap_linesF_allocd = 0; // # LineF's issued 563ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__secmap_linesF_bytes = 0; // .. using this much storage 564ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__secmap_iterator_steppings = 0; // # calls to stepSMIter 565ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cache_Z_fetches = 0; // # Z lines fetched 566ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cache_Z_wbacks = 0; // # Z lines written back 567ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cache_F_fetches = 0; // # F lines fetched 568ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cache_F_wbacks = 0; // # F lines written back 569ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cache_invals = 0; // # cache invals 570ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cache_flushes = 0; // # cache flushes 571ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cache_totrefs = 0; // # total accesses 572ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cache_totmisses = 0; // # misses 573ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic ULong stats__cache_make_New_arange = 0; // total arange made New 574ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic ULong stats__cache_make_New_inZrep = 0; // arange New'd on Z reps 575ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_normalises = 0; // # calls to cacheline_normalise 576ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_cread64s = 0; // # calls to s_m_read64 577ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_cread32s = 0; // # calls to s_m_read32 578ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_cread16s = 0; // # calls to s_m_read16 579ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_cread08s = 0; // # calls to s_m_read8 580ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_cwrite64s = 0; // # calls to s_m_write64 581ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_cwrite32s = 0; // # calls to s_m_write32 582ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_cwrite16s = 0; // # calls to s_m_write16 583ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_cwrite08s = 0; // # calls to s_m_write8 584ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_sread08s = 0; // # calls to s_m_set8 585ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_swrite08s = 0; // # calls to s_m_get8 586ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_swrite16s = 0; // # calls to s_m_get8 587ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_swrite32s = 0; // # calls to s_m_get8 588ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_swrite64s = 0; // # calls to s_m_get8 589ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_scopy08s = 0; // # calls to s_m_copy8 590ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_64to32splits = 0; // # 64-bit accesses split 591ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_32to16splits = 0; // # 32-bit accesses split 592ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_16to8splits = 0; // # 16-bit accesses split 593ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_64to32pulldown = 0; // # calls to pulldown_to_32 594ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_32to16pulldown = 0; // # calls to pulldown_to_16 595ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__cline_16to8pulldown = 0; // # calls to pulldown_to_8 596ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__vts__tick = 0; // # calls to VTS__tick 597ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__vts__join = 0; // # calls to VTS__join 598ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__vts__cmpLEQ = 0; // # calls to VTS__cmpLEQ 599ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__vts__cmp_structural = 0; // # calls to VTS__cmp_structural 600b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 601b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// # calls to VTS__cmp_structural w/ slow case 602b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic UWord stats__vts__cmp_structural_slow = 0; 603b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 604b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// # calls to VTS__indexAt_SLOW 605b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic UWord stats__vts__indexat_slow = 0; 606b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 607b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// # calls to vts_set__find__or__clone_and_add 608b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic UWord stats__vts_set__focaa = 0; 609b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 610b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// # calls to vts_set__find__or__clone_and_add that lead to an 611b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// allocation 612b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic UWord stats__vts_set__focaa_a = 0; 613ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 614ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 615ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Addr shmem__round_to_SecMap_base ( Addr a ) { 616ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return a & ~(N_SECMAP_ARANGE - 1); 617ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 618ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline UWord shmem__get_SecMap_offset ( Addr a ) { 619ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return a & (N_SECMAP_ARANGE - 1); 620ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 621ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 622ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 623ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*----------------------------------------------------------------*/ 624ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--- map_shmem :: WordFM Addr SecMap ---*/ 625ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--- shadow memory (low level handlers) (shmem__* fns) ---*/ 626ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*----------------------------------------------------------------*/ 627ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 628ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--------------- SecMap allocation --------------- */ 629ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 630ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic HChar* shmem__bigchunk_next = NULL; 631ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic HChar* shmem__bigchunk_end1 = NULL; 632ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 633ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void* shmem__bigchunk_alloc ( SizeT n ) 634ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 635ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown const SizeT sHMEM__BIGCHUNK_SIZE = 4096 * 256 * 4; 636ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(n > 0); 637ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown n = VG_ROUNDUP(n, 16); 638ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(shmem__bigchunk_next <= shmem__bigchunk_end1); 639ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(shmem__bigchunk_end1 - shmem__bigchunk_next 640ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown <= (SSizeT)sHMEM__BIGCHUNK_SIZE); 641ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (shmem__bigchunk_next + n > shmem__bigchunk_end1) { 642ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) 643ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("XXXXX bigchunk: abandoning %d bytes\n", 644ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)(shmem__bigchunk_end1 - shmem__bigchunk_next)); 645ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown shmem__bigchunk_next = VG_(am_shadow_alloc)( sHMEM__BIGCHUNK_SIZE ); 646ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (shmem__bigchunk_next == NULL) 647ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(out_of_memory_NORETURN)( 648ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "helgrind:shmem__bigchunk_alloc", sHMEM__BIGCHUNK_SIZE ); 649ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown shmem__bigchunk_end1 = shmem__bigchunk_next + sHMEM__BIGCHUNK_SIZE; 650ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 651ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(shmem__bigchunk_next); 652ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert( 0 == (((Addr)shmem__bigchunk_next) & (16-1)) ); 653ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(shmem__bigchunk_next + n <= shmem__bigchunk_end1); 654ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown shmem__bigchunk_next += n; 655ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return shmem__bigchunk_next - n; 656ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 657ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 658ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic SecMap* shmem__alloc_SecMap ( void ) 659ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 660ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i, j; 661ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SecMap* sm = shmem__bigchunk_alloc( sizeof(SecMap) ); 662ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)("alloc_SecMap %p\n",sm); 663ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm); 664ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sm->magic = SecMap_MAGIC; 665ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_SECMAP_ZLINES; i++) { 666ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sm->linesZ[i].dict[0] = SVal_NOACCESS; 667ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sm->linesZ[i].dict[1] = SVal_INVALID; 668ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sm->linesZ[i].dict[2] = SVal_INVALID; 669ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sm->linesZ[i].dict[3] = SVal_INVALID; 670ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (j = 0; j < N_LINE_ARANGE/4; j++) 671ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sm->linesZ[i].ix2s[j] = 0; /* all reference dict[0] */ 672ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 673ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sm->linesF = NULL; 674ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sm->linesF_size = 0; 675ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmaps_allocd++; 676ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmap_ga_space_covered += N_SECMAP_ARANGE; 677ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmap_linesZ_allocd += N_SECMAP_ZLINES; 678ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmap_linesZ_bytes += N_SECMAP_ZLINES * sizeof(LineZ); 679ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return sm; 680ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 681ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 682ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef struct { Addr gaKey; SecMap* sm; } SMCacheEnt; 683ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic SMCacheEnt smCache[3] = { {1,NULL}, {1,NULL}, {1,NULL} }; 684ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 685ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic SecMap* shmem__find_SecMap ( Addr ga ) 686ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 687ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SecMap* sm = NULL; 688ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr gaKey = shmem__round_to_SecMap_base(ga); 689ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // Cache 690ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmaps_search++; 691ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(gaKey == smCache[0].gaKey)) 692ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return smCache[0].sm; 693ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(gaKey == smCache[1].gaKey)) { 694ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SMCacheEnt tmp = smCache[0]; 695ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown smCache[0] = smCache[1]; 696ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown smCache[1] = tmp; 697ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return smCache[0].sm; 698ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 699ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (gaKey == smCache[2].gaKey) { 700ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SMCacheEnt tmp = smCache[1]; 701ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown smCache[1] = smCache[2]; 702ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown smCache[2] = tmp; 703ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return smCache[1].sm; 704ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 705ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // end Cache 706ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmaps_search_slow++; 707ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (VG_(lookupFM)( map_shmem, 708ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown NULL/*keyP*/, (UWord*)&sm, (UWord)gaKey )) { 709ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm != NULL); 710ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown smCache[2] = smCache[1]; 711ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown smCache[1] = smCache[0]; 712ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown smCache[0].gaKey = gaKey; 713ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown smCache[0].sm = sm; 714ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 715ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm == NULL); 716ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 717ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return sm; 718ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 719ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 720ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic SecMap* shmem__find_or_alloc_SecMap ( Addr ga ) 721ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 722ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SecMap* sm = shmem__find_SecMap ( ga ); 723ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(sm)) { 724ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return sm; 725ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 726ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* create a new one */ 727ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr gaKey = shmem__round_to_SecMap_base(ga); 728ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sm = shmem__alloc_SecMap(); 729ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm); 730ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(addToFM)( map_shmem, (UWord)gaKey, (UWord)sm ); 731ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return sm; 732ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 733ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 734ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 735ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 736ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* ------------ LineF and LineZ related ------------ */ 737ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 738ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void rcinc_LineF ( LineF* lineF ) { 739ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i; 740ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(lineF->inUse); 741ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_LINE_ARANGE; i++) 742ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcinc(lineF->w64s[i]); 743ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 744ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 745ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void rcdec_LineF ( LineF* lineF ) { 746ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i; 747ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(lineF->inUse); 748ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_LINE_ARANGE; i++) 749ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcdec(lineF->w64s[i]); 750ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 751ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 752ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void rcinc_LineZ ( LineZ* lineZ ) { 753ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(lineZ->dict[0] != SVal_INVALID); 754ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcinc(lineZ->dict[0]); 755ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lineZ->dict[1] != SVal_INVALID) rcinc(lineZ->dict[1]); 756ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lineZ->dict[2] != SVal_INVALID) rcinc(lineZ->dict[2]); 757ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lineZ->dict[3] != SVal_INVALID) rcinc(lineZ->dict[3]); 758ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 759ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 760ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void rcdec_LineZ ( LineZ* lineZ ) { 761ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(lineZ->dict[0] != SVal_INVALID); 762ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcdec(lineZ->dict[0]); 763ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lineZ->dict[1] != SVal_INVALID) rcdec(lineZ->dict[1]); 764ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lineZ->dict[2] != SVal_INVALID) rcdec(lineZ->dict[2]); 765ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lineZ->dict[3] != SVal_INVALID) rcdec(lineZ->dict[3]); 766ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 767ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 768ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browninline 769ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void write_twobit_array ( UChar* arr, UWord ix, UWord b2 ) { 770ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word bix, shft, mask, prep; 771ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ix >= 0); 772ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown bix = ix >> 2; 773ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown shft = 2 * (ix & 3); /* 0, 2, 4 or 6 */ 774ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mask = 3 << shft; 775ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown prep = b2 << shft; 776ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown arr[bix] = (arr[bix] & ~mask) | prep; 777ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 778ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 779ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browninline 780ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord read_twobit_array ( UChar* arr, UWord ix ) { 781ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word bix, shft; 782ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ix >= 0); 783ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown bix = ix >> 2; 784ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown shft = 2 * (ix & 3); /* 0, 2, 4 or 6 */ 785ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return (arr[bix] >> shft) & 3; 786ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 787ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 788ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Given address 'tag', find either the Z or F line containing relevant 789ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown data, so it can be read into the cache. 790ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 791ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void find_ZF_for_reading ( /*OUT*/LineZ** zp, 792ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /*OUT*/LineF** fp, Addr tag ) { 793ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineZ* lineZ; 794ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineF* lineF; 795ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord zix; 796ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SecMap* sm = shmem__find_or_alloc_SecMap(tag); 797ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord smoff = shmem__get_SecMap_offset(tag); 798ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* since smoff is derived from a valid tag, it should be 799ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cacheline-aligned. */ 800ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0 == (smoff & (N_LINE_ARANGE - 1))); 801ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zix = smoff >> N_LINE_BITS; 802ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(zix < N_SECMAP_ZLINES); 803ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineZ = &sm->linesZ[zix]; 804ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineF = NULL; 805ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lineZ->dict[0] == SVal_INVALID) { 806ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt fix = (UInt)lineZ->dict[1]; 807ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm->linesF); 808ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm->linesF_size > 0); 809ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(fix >= 0 && fix < sm->linesF_size); 810ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineF = &sm->linesF[fix]; 811ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(lineF->inUse); 812ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineZ = NULL; 813ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 814ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *zp = lineZ; 815ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *fp = lineF; 816ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 817ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 818ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Given address 'tag', return the relevant SecMap and the index of 819ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown the LineZ within it, in the expectation that the line is to be 820ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown overwritten. Regardless of whether 'tag' is currently associated 821ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown with a Z or F representation, to rcdec on the current 822ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown representation, in recognition of the fact that the contents are 823ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown just about to be overwritten. */ 824ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __attribute__((noinline)) 825ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid find_Z_for_writing ( /*OUT*/SecMap** smp, 826ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /*OUT*/Word* zixp, 827ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr tag ) { 828ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineZ* lineZ; 829ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineF* lineF; 830ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord zix; 831ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SecMap* sm = shmem__find_or_alloc_SecMap(tag); 832ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord smoff = shmem__get_SecMap_offset(tag); 833ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* since smoff is derived from a valid tag, it should be 834ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cacheline-aligned. */ 835ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0 == (smoff & (N_LINE_ARANGE - 1))); 836ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zix = smoff >> N_LINE_BITS; 837ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(zix < N_SECMAP_ZLINES); 838ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineZ = &sm->linesZ[zix]; 839ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineF = NULL; 840ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* re RCs, we are freeing up this LineZ/LineF so that new data can 841ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown be parked in it. Hence have to rcdec it accordingly. */ 842ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* If lineZ has an associated lineF, free it up. */ 843ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lineZ->dict[0] == SVal_INVALID) { 844ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt fix = (UInt)lineZ->dict[1]; 845ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm->linesF); 846ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm->linesF_size > 0); 847ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(fix >= 0 && fix < sm->linesF_size); 848ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineF = &sm->linesF[fix]; 849ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(lineF->inUse); 850ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcdec_LineF(lineF); 851ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineF->inUse = False; 852ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 853ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcdec_LineZ(lineZ); 854ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 855ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *smp = sm; 856ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *zixp = zix; 857ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 858ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 859ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __attribute__((noinline)) 860ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid alloc_F_for_writing ( /*MOD*/SecMap* sm, /*OUT*/Word* fixp ) { 861ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt i, new_size; 862ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineF* nyu; 863ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 864ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (sm->linesF) { 865ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm->linesF_size > 0); 866ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 867ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm->linesF_size == 0); 868ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 869ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 870ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (sm->linesF) { 871ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < sm->linesF_size; i++) { 872ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!sm->linesF[i].inUse) { 873ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *fixp = (Word)i; 874ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 875ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 876ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 877ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 878ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 879ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* No free F line found. Expand existing array and try again. */ 880ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown new_size = sm->linesF_size==0 ? 1 : 2 * sm->linesF_size; 881ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nyu = HG_(zalloc)( "libhb.aFfw.1 (LineF storage)", 882ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown new_size * sizeof(LineF) ); 883ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(nyu); 884ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 885ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmap_linesF_allocd += (new_size - sm->linesF_size); 886ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmap_linesF_bytes += (new_size - sm->linesF_size) 887ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown * sizeof(LineF); 888ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 889ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) 890ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("SM %p: expand F array from %d to %d\n", 891ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sm, (Int)sm->linesF_size, new_size); 892ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 893ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < new_size; i++) 894ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nyu[i].inUse = False; 895ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 896ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (sm->linesF) { 897ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < sm->linesF_size; i++) { 898ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm->linesF[i].inUse); 899ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nyu[i] = sm->linesF[i]; 900ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 901ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(memset)(sm->linesF, 0, sm->linesF_size * sizeof(LineF) ); 902ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free)(sm->linesF); 903ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 904ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 905ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sm->linesF = nyu; 906ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sm->linesF_size = new_size; 907ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 908ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < sm->linesF_size; i++) { 909ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!sm->linesF[i].inUse) { 910ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *fixp = (Word)i; 911ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 912ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 913ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 914ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 915ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /*NOTREACHED*/ 916ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 917ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 918ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 919ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 920ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* ------------ CacheLine and implicit-tree related ------------ */ 921ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 922ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown__attribute__((unused)) 923ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void pp_CacheLine ( CacheLine* cl ) { 924ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i; 925ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!cl) { 926ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","pp_CacheLine(NULL)\n"); 927ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 928ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 929ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_LINE_TREES; i++) 930ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" descr: %04lx\n", (UWord)cl->descrs[i]); 931ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_LINE_ARANGE; i++) 932ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" sval: %08lx\n", (UWord)cl->svals[i]); 933ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 934ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 935ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UChar descr_to_validbits ( UShort descr ) 936ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 937ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* a.k.a Party Time for gcc's constant folder */ 938ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# define DESCR(b8_7, b8_6, b8_5, b8_4, b8_3, b8_2, b8_1, b8_0, \ 939ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b16_3, b32_1, b16_2, b64, b16_1, b32_0, b16_0) \ 940ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ( (UShort) ( ( (b8_7) << 14) | ( (b8_6) << 13) | \ 941ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ( (b8_5) << 12) | ( (b8_4) << 11) | \ 942ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ( (b8_3) << 10) | ( (b8_2) << 9) | \ 943ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ( (b8_1) << 8) | ( (b8_0) << 7) | \ 944ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ( (b16_3) << 6) | ( (b32_1) << 5) | \ 945ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ( (b16_2) << 4) | ( (b64) << 3) | \ 946ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ( (b16_1) << 2) | ( (b32_0) << 1) | \ 947ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ( (b16_0) << 0) ) ) 948ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 949ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# define BYTE(bit7, bit6, bit5, bit4, bit3, bit2, bit1, bit0) \ 950ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ( (UChar) ( ( (bit7) << 7) | ( (bit6) << 6) | \ 951ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ( (bit5) << 5) | ( (bit4) << 4) | \ 952ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ( (bit3) << 3) | ( (bit2) << 2) | \ 953ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ( (bit1) << 1) | ( (bit0) << 0) ) ) 954ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 955ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* these should all get folded out at compile time */ 956ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(DESCR(1,0,0,0,0,0,0,0, 0,0,0, 0, 0,0,0) == TREE_DESCR_8_7); 957ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(DESCR(0,0,0,0,0,0,0,1, 0,0,0, 0, 0,0,0) == TREE_DESCR_8_0); 958ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(DESCR(0,0,0,0,0,0,0,0, 1,0,0, 0, 0,0,0) == TREE_DESCR_16_3); 959ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(DESCR(0,0,0,0,0,0,0,0, 0,1,0, 0, 0,0,0) == TREE_DESCR_32_1); 960ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(DESCR(0,0,0,0,0,0,0,0, 0,0,1, 0, 0,0,0) == TREE_DESCR_16_2); 961ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(DESCR(0,0,0,0,0,0,0,0, 0,0,0, 1, 0,0,0) == TREE_DESCR_64); 962ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(DESCR(0,0,0,0,0,0,0,0, 0,0,0, 0, 1,0,0) == TREE_DESCR_16_1); 963ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(DESCR(0,0,0,0,0,0,0,0, 0,0,0, 0, 0,1,0) == TREE_DESCR_32_0); 964ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(DESCR(0,0,0,0,0,0,0,0, 0,0,0, 0, 0,0,1) == TREE_DESCR_16_0); 965ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 966ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown switch (descr) { 967ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* 968ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown +--------------------------------- TREE_DESCR_8_7 969ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | +------------------- TREE_DESCR_8_0 970ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | | +---------------- TREE_DESCR_16_3 971ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | | | +-------------- TREE_DESCR_32_1 972ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | | | | +------------ TREE_DESCR_16_2 973ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | | | | | +--------- TREE_DESCR_64 974ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | | | | | | +------ TREE_DESCR_16_1 975ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | | | | | | | +---- TREE_DESCR_32_0 976ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | | | | | | | | +-- TREE_DESCR_16_0 977ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | | | | | | | | | 978ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | | | | | | | | | GRANULARITY, 7 -> 0 */ 979ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(1,1,1,1,1,1,1,1, 0,0,0, 0, 0,0,0): /* 8 8 8 8 8 8 8 8 */ 980ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(1,1,1,1,1,1,1,1); 981ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(1,1,0,0,1,1,1,1, 0,0,1, 0, 0,0,0): /* 8 8 16 8 8 8 8 */ 982ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(1,1,0,1,1,1,1,1); 983ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,1,1,1,1,1,1, 1,0,0, 0, 0,0,0): /* 16 8 8 8 8 8 8 */ 984ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,1,1,1,1,1,1,1); 985ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,0,0,1,1,1,1, 1,0,1, 0, 0,0,0): /* 16 16 8 8 8 8 */ 986ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,1,0,1,1,1,1,1); 987ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 988ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(1,1,1,1,1,1,0,0, 0,0,0, 0, 0,0,1): /* 8 8 8 8 8 8 16 */ 989ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(1,1,1,1,1,1,0,1); 990ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(1,1,0,0,1,1,0,0, 0,0,1, 0, 0,0,1): /* 8 8 16 8 8 16 */ 991ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(1,1,0,1,1,1,0,1); 992ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,1,1,1,1,0,0, 1,0,0, 0, 0,0,1): /* 16 8 8 8 8 16 */ 993ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,1,1,1,1,1,0,1); 994ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,0,0,1,1,0,0, 1,0,1, 0, 0,0,1): /* 16 16 8 8 16 */ 995ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,1,0,1,1,1,0,1); 996ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 997ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(1,1,1,1,0,0,1,1, 0,0,0, 0, 1,0,0): /* 8 8 8 8 16 8 8 */ 998ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(1,1,1,1,0,1,1,1); 999ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(1,1,0,0,0,0,1,1, 0,0,1, 0, 1,0,0): /* 8 8 16 16 8 8 */ 1000ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(1,1,0,1,0,1,1,1); 1001ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,1,1,0,0,1,1, 1,0,0, 0, 1,0,0): /* 16 8 8 16 8 8 */ 1002ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,1,1,1,0,1,1,1); 1003ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,0,0,0,0,1,1, 1,0,1, 0, 1,0,0): /* 16 16 16 8 8 */ 1004ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,1,0,1,0,1,1,1); 1005ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1006ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(1,1,1,1,0,0,0,0, 0,0,0, 0, 1,0,1): /* 8 8 8 8 16 16 */ 1007ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(1,1,1,1,0,1,0,1); 1008ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(1,1,0,0,0,0,0,0, 0,0,1, 0, 1,0,1): /* 8 8 16 16 16 */ 1009ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(1,1,0,1,0,1,0,1); 1010ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,1,1,0,0,0,0, 1,0,0, 0, 1,0,1): /* 16 8 8 16 16 */ 1011ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,1,1,1,0,1,0,1); 1012ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,0,0,0,0,0,0, 1,0,1, 0, 1,0,1): /* 16 16 16 16 */ 1013ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,1,0,1,0,1,0,1); 1014ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1015ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,0,0,1,1,1,1, 0,1,0, 0, 0,0,0): /* 32 8 8 8 8 */ 1016ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,0,0,1,1,1,1,1); 1017ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,0,0,1,1,0,0, 0,1,0, 0, 0,0,1): /* 32 8 8 16 */ 1018ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,0,0,1,1,1,0,1); 1019ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,0,0,0,0,1,1, 0,1,0, 0, 1,0,0): /* 32 16 8 8 */ 1020ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,0,0,1,0,1,1,1); 1021ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,0,0,0,0,0,0, 0,1,0, 0, 1,0,1): /* 32 16 16 */ 1022ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,0,0,1,0,1,0,1); 1023ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1024ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(1,1,1,1,0,0,0,0, 0,0,0, 0, 0,1,0): /* 8 8 8 8 32 */ 1025ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(1,1,1,1,0,0,0,1); 1026ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(1,1,0,0,0,0,0,0, 0,0,1, 0, 0,1,0): /* 8 8 16 32 */ 1027ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(1,1,0,1,0,0,0,1); 1028ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,1,1,0,0,0,0, 1,0,0, 0, 0,1,0): /* 16 8 8 32 */ 1029ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,1,1,1,0,0,0,1); 1030ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,0,0,0,0,0,0, 1,0,1, 0, 0,1,0): /* 16 16 32 */ 1031ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,1,0,1,0,0,0,1); 1032ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1033ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,0,0,0,0,0,0, 0,1,0, 0, 0,1,0): /* 32 32 */ 1034ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,0,0,1,0,0,0,1); 1035ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1036ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case DESCR(0,0,0,0,0,0,0,0, 0,0,0, 1, 0,0,0): /* 64 */ 1037ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return BYTE(0,0,0,0,0,0,0,1); 1038ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1039ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown default: return BYTE(0,0,0,0,0,0,0,0); 1040ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* INVALID - any valid descr produces at least one 1041ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown valid bit in tree[0..7]*/ 1042ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1043ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* NOTREACHED*/ 1044ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 1045ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1046ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# undef DESCR 1047ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# undef BYTE 1048ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1049ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1050ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown__attribute__((unused)) 1051ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Bool is_sane_Descr ( UShort descr ) { 1052ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return descr_to_validbits(descr) != 0; 1053ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1054ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1055ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void sprintf_Descr ( /*OUT*/HChar* dst, UShort descr ) { 1056ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(sprintf)(dst, 1057ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "%d%d%d%d%d%d%d%d %d%d%d %d %d%d%d", 1058ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_8_7) ? 1 : 0), 1059ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_8_6) ? 1 : 0), 1060ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_8_5) ? 1 : 0), 1061ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_8_4) ? 1 : 0), 1062ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_8_3) ? 1 : 0), 1063ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_8_2) ? 1 : 0), 1064ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_8_1) ? 1 : 0), 1065ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_8_0) ? 1 : 0), 1066ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_16_3) ? 1 : 0), 1067ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_32_1) ? 1 : 0), 1068ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_16_2) ? 1 : 0), 1069ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_64) ? 1 : 0), 1070ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_16_1) ? 1 : 0), 1071ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_32_0) ? 1 : 0), 1072ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((descr & TREE_DESCR_16_0) ? 1 : 0) 1073ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ); 1074ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1075ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void sprintf_Byte ( /*OUT*/HChar* dst, UChar byte ) { 1076ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(sprintf)(dst, "%d%d%d%d%d%d%d%d", 1077ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((byte & 128) ? 1 : 0), 1078ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((byte & 64) ? 1 : 0), 1079ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((byte & 32) ? 1 : 0), 1080ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((byte & 16) ? 1 : 0), 1081ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((byte & 8) ? 1 : 0), 1082ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((byte & 4) ? 1 : 0), 1083ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((byte & 2) ? 1 : 0), 1084ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)((byte & 1) ? 1 : 0) 1085ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ); 1086ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1087ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1088ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Bool is_sane_Descr_and_Tree ( UShort descr, SVal* tree ) { 1089ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i; 1090ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UChar validbits = descr_to_validbits(descr); 1091ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HChar buf[128], buf2[128]; 1092ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (validbits == 0) 1093ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto bad; 1094ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < 8; i++) { 1095ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (validbits & (1<<i)) { 1096ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tree[i] == SVal_INVALID) 1097ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto bad; 1098ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 1099ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tree[i] != SVal_INVALID) 1100ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto bad; 1101ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1102ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1103ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return True; 1104ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown bad: 1105ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sprintf_Descr( buf, descr ); 1106ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sprintf_Byte( buf2, validbits ); 1107ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","is_sane_Descr_and_Tree: bad tree {\n"); 1108ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" validbits 0x%02lx %s\n", (UWord)validbits, buf2); 1109ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" descr 0x%04lx %s\n", (UWord)descr, buf); 1110ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < 8; i++) 1111ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" [%ld] 0x%016llx\n", i, tree[i]); 1112ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","}\n"); 1113ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0; 1114ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1115ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1116ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Bool is_sane_CacheLine ( CacheLine* cl ) 1117ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1118ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word tno, cloff; 1119ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1120ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!cl) goto bad; 1121ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1122ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (tno = 0, cloff = 0; tno < N_LINE_TREES; tno++, cloff += 8) { 1123ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr = cl->descrs[tno]; 1124ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal* tree = &cl->svals[cloff]; 1125ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!is_sane_Descr_and_Tree(descr, tree)) 1126ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto bad; 1127ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1128ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(cloff == N_LINE_ARANGE); 1129ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return True; 1130ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown bad: 1131ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown pp_CacheLine(cl); 1132ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 1133ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1134ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1135ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UShort normalise_tree ( /*MOD*/SVal* tree ) 1136ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1137ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr; 1138ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* pre: incoming tree[0..7] does not have any invalid shvals, in 1139ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown particular no zeroes. */ 1140ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(tree[7] == SVal_INVALID || tree[6] == SVal_INVALID 1141ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown || tree[5] == SVal_INVALID || tree[4] == SVal_INVALID 1142ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown || tree[3] == SVal_INVALID || tree[2] == SVal_INVALID 1143ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown || tree[1] == SVal_INVALID || tree[0] == SVal_INVALID)) 1144ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 1145ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1146ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = TREE_DESCR_8_7 | TREE_DESCR_8_6 | TREE_DESCR_8_5 1147ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | TREE_DESCR_8_4 | TREE_DESCR_8_3 | TREE_DESCR_8_2 1148ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | TREE_DESCR_8_1 | TREE_DESCR_8_0; 1149ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* build 16-bit layer */ 1150ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tree[1] == tree[0]) { 1151ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[1] = SVal_INVALID; 1152ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~(TREE_DESCR_8_1 | TREE_DESCR_8_0); 1153ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= TREE_DESCR_16_0; 1154ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1155ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tree[3] == tree[2]) { 1156ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[3] = SVal_INVALID; 1157ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~(TREE_DESCR_8_3 | TREE_DESCR_8_2); 1158ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= TREE_DESCR_16_1; 1159ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1160ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tree[5] == tree[4]) { 1161ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[5] = SVal_INVALID; 1162ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~(TREE_DESCR_8_5 | TREE_DESCR_8_4); 1163ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= TREE_DESCR_16_2; 1164ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1165ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tree[7] == tree[6]) { 1166ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[7] = SVal_INVALID; 1167ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~(TREE_DESCR_8_7 | TREE_DESCR_8_6); 1168ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= TREE_DESCR_16_3; 1169ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1170ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* build 32-bit layer */ 1171ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tree[2] == tree[0] 1172ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown && (descr & TREE_DESCR_16_1) && (descr & TREE_DESCR_16_0)) { 1173ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[2] = SVal_INVALID; /* [3,1] must already be SVal_INVALID */ 1174ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~(TREE_DESCR_16_1 | TREE_DESCR_16_0); 1175ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= TREE_DESCR_32_0; 1176ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1177ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tree[6] == tree[4] 1178ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown && (descr & TREE_DESCR_16_3) && (descr & TREE_DESCR_16_2)) { 1179ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[6] = SVal_INVALID; /* [7,5] must already be SVal_INVALID */ 1180ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~(TREE_DESCR_16_3 | TREE_DESCR_16_2); 1181ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= TREE_DESCR_32_1; 1182ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1183ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* build 64-bit layer */ 1184ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tree[4] == tree[0] 1185ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown && (descr & TREE_DESCR_32_1) && (descr & TREE_DESCR_32_0)) { 1186ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[4] = SVal_INVALID; /* [7,6,5,3,2,1] must already be SVal_INVALID */ 1187ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~(TREE_DESCR_32_1 | TREE_DESCR_32_0); 1188ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= TREE_DESCR_64; 1189ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1190ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return descr; 1191ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1192ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1193ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* This takes a cacheline where all the data is at the leaves 1194ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (w8[..]) and builds a correctly normalised tree. */ 1195ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void normalise_CacheLine ( /*MOD*/CacheLine* cl ) 1196ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1197ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word tno, cloff; 1198ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (tno = 0, cloff = 0; tno < N_LINE_TREES; tno++, cloff += 8) { 1199ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal* tree = &cl->svals[cloff]; 1200ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = normalise_tree( tree ); 1201ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1202ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(cloff == N_LINE_ARANGE); 1203ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 1204ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */ 1205ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_normalises++; 1206ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1207ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1208ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1209ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef struct { UChar count; SVal sval; } CountedSVal; 1210ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1211ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 1212ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid sequentialise_CacheLine ( /*OUT*/CountedSVal* dst, 1213ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /*OUT*/Word* dstUsedP, 1214ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word nDst, CacheLine* src ) 1215ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1216ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word tno, cloff, dstUsed; 1217ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1218ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(nDst == N_LINE_ARANGE); 1219ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown dstUsed = 0; 1220ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1221ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (tno = 0, cloff = 0; tno < N_LINE_TREES; tno++, cloff += 8) { 1222ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr = src->descrs[tno]; 1223ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal* tree = &src->svals[cloff]; 1224ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1225ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* sequentialise the tree described by (descr,tree). */ 1226ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# define PUT(_n,_v) \ 1227ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown do { dst[dstUsed ].count = (_n); \ 1228ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown dst[dstUsed++].sval = (_v); \ 1229ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } while (0) 1230ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1231ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* byte 0 */ 1232ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_64) PUT(8, tree[0]); else 1233ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_32_0) PUT(4, tree[0]); else 1234ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_16_0) PUT(2, tree[0]); else 1235ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_8_0) PUT(1, tree[0]); 1236ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* byte 1 */ 1237ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_8_1) PUT(1, tree[1]); 1238ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* byte 2 */ 1239ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_16_1) PUT(2, tree[2]); else 1240ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_8_2) PUT(1, tree[2]); 1241ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* byte 3 */ 1242ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_8_3) PUT(1, tree[3]); 1243ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* byte 4 */ 1244ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_32_1) PUT(4, tree[4]); else 1245ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_16_2) PUT(2, tree[4]); else 1246ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_8_4) PUT(1, tree[4]); 1247ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* byte 5 */ 1248ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_8_5) PUT(1, tree[5]); 1249ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* byte 6 */ 1250ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_16_3) PUT(2, tree[6]); else 1251ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_8_6) PUT(1, tree[6]); 1252ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* byte 7 */ 1253ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (descr & TREE_DESCR_8_7) PUT(1, tree[7]); 1254ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1255ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# undef PUT 1256ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* END sequentialise the tree described by (descr,tree). */ 1257ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1258ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1259ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(cloff == N_LINE_ARANGE); 1260ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(dstUsed <= nDst); 1261ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1262ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *dstUsedP = dstUsed; 1263ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1264ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1265ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Write the cacheline 'wix' to backing store. Where it ends up 1266ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown is determined by its tag field. */ 1267ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __attribute__((noinline)) void cacheline_wback ( UWord wix ) 1268ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1269ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i, j, k, m; 1270ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr tag; 1271ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SecMap* sm; 1272ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 1273ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineZ* lineZ; 1274ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineF* lineF; 1275ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word zix, fix, csvalsUsed; 1276ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CountedSVal csvals[N_LINE_ARANGE]; 1277ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal sv; 1278ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1279ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) 1280ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("scache wback line %d\n", (Int)wix); 1281ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1282ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(wix >= 0 && wix < N_WAY_NENT); 1283ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1284ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tag = cache_shmem.tags0[wix]; 1285ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = &cache_shmem.lyns0[wix]; 1286ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1287ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* The cache line may have been invalidated; if so, ignore it. */ 1288ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!is_valid_scache_tag(tag)) 1289ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 1290ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1291ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Where are we going to put it? */ 1292ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sm = NULL; 1293ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineZ = NULL; 1294ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineF = NULL; 1295ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zix = fix = -1; 1296ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1297ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* find the Z line to write in and rcdec it or the associated F 1298ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line. */ 1299ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown find_Z_for_writing( &sm, &zix, tag ); 1300ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1301ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm); 1302ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(zix >= 0 && zix < N_SECMAP_ZLINES); 1303ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineZ = &sm->linesZ[zix]; 1304ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1305ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Generate the data to be stored */ 1306ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 1307ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */ 1308ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1309ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown csvalsUsed = -1; 1310ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sequentialise_CacheLine( csvals, &csvalsUsed, 1311ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown N_LINE_ARANGE, cl ); 1312ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(csvalsUsed >= 1 && csvalsUsed <= N_LINE_ARANGE); 1313ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)("%lu ", csvalsUsed); 1314ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1315ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineZ->dict[0] = lineZ->dict[1] 1316ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown = lineZ->dict[2] = lineZ->dict[3] = SVal_INVALID; 1317ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1318ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* i indexes actual shadow values, k is cursor in csvals */ 1319ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown i = 0; 1320ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (k = 0; k < csvalsUsed; k++) { 1321ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1322ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sv = csvals[k].sval; 1323ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 1324ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(csvals[k].count >= 1 && csvals[k].count <= 8); 1325ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* do we already have it? */ 1326ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (sv == lineZ->dict[0]) { j = 0; goto dict_ok; } 1327ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (sv == lineZ->dict[1]) { j = 1; goto dict_ok; } 1328ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (sv == lineZ->dict[2]) { j = 2; goto dict_ok; } 1329ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (sv == lineZ->dict[3]) { j = 3; goto dict_ok; } 1330ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* no. look for a free slot. */ 1331ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 1332ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sv != SVal_INVALID); 1333ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lineZ->dict[0] 1334ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown == SVal_INVALID) { lineZ->dict[0] = sv; j = 0; goto dict_ok; } 1335ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lineZ->dict[1] 1336ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown == SVal_INVALID) { lineZ->dict[1] = sv; j = 1; goto dict_ok; } 1337ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lineZ->dict[2] 1338ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown == SVal_INVALID) { lineZ->dict[2] = sv; j = 2; goto dict_ok; } 1339ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lineZ->dict[3] 1340ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown == SVal_INVALID) { lineZ->dict[3] = sv; j = 3; goto dict_ok; } 1341ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; /* we'll have to use the f rep */ 1342ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown dict_ok: 1343ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown m = csvals[k].count; 1344ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (m == 8) { 1345ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+0, j ); 1346ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+1, j ); 1347ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+2, j ); 1348ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+3, j ); 1349ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+4, j ); 1350ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+5, j ); 1351ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+6, j ); 1352ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+7, j ); 1353ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown i += 8; 1354ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1355ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown else if (m == 4) { 1356ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+0, j ); 1357ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+1, j ); 1358ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+2, j ); 1359ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+3, j ); 1360ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown i += 4; 1361ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1362ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown else if (m == 1) { 1363ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+0, j ); 1364ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown i += 1; 1365ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1366ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown else if (m == 2) { 1367ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+0, j ); 1368ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_twobit_array( lineZ->ix2s, i+1, j ); 1369ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown i += 2; 1370ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1371ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown else { 1372ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); /* 8 4 2 or 1 are the only legitimate values for m */ 1373ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1374ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1375ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1376ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1377ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(i == N_LINE_ARANGE)) { 1378ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Construction of the compressed representation was 1379ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown successful. */ 1380ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcinc_LineZ(lineZ); 1381ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_Z_wbacks++; 1382ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 1383ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Cannot use the compressed(z) representation. Use the full(f) 1384ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rep instead. */ 1385ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(i >= 0 && i < N_LINE_ARANGE); 1386ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown alloc_F_for_writing( sm, &fix ); 1387ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm->linesF); 1388ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm->linesF_size > 0); 1389ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(fix >= 0 && fix < (Word)sm->linesF_size); 1390ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineF = &sm->linesF[fix]; 1391ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(!lineF->inUse); 1392ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineZ->dict[0] = lineZ->dict[2] = lineZ->dict[3] = SVal_INVALID; 1393ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineZ->dict[1] = (SVal)fix; 1394ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineF->inUse = True; 1395ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown i = 0; 1396ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (k = 0; k < csvalsUsed; k++) { 1397ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 1398ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(csvals[k].count >= 1 && csvals[k].count <= 8); 1399ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sv = csvals[k].sval; 1400ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 1401ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sv != SVal_INVALID); 1402ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (m = csvals[k].count; m > 0; m--) { 1403ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineF->w64s[i] = sv; 1404ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown i++; 1405ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1406ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1407ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(i == N_LINE_ARANGE); 1408ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcinc_LineF(lineF); 1409ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_F_wbacks++; 1410ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1411ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1412ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1413ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Fetch the cacheline 'wix' from the backing store. The tag 1414ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown associated with 'wix' is assumed to have already been filled in; 1415ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown hence that is used to determine where in the backing store to read 1416ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown from. */ 1417ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __attribute__((noinline)) void cacheline_fetch ( UWord wix ) 1418ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1419ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i; 1420ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr tag; 1421ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 1422ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineZ* lineZ; 1423ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineF* lineF; 1424ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1425ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) 1426ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("scache fetch line %d\n", (Int)wix); 1427ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1428ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(wix >= 0 && wix < N_WAY_NENT); 1429ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1430ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tag = cache_shmem.tags0[wix]; 1431ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = &cache_shmem.lyns0[wix]; 1432ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1433ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* reject nonsense requests */ 1434ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_valid_scache_tag(tag)); 1435ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1436ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineZ = NULL; 1437ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineF = NULL; 1438ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown find_ZF_for_reading( &lineZ, &lineF, tag ); 1439ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert( (lineZ && !lineF) || (!lineZ && lineF) ); 1440ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1441ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* expand the data into the bottom layer of the tree, then get 1442ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cacheline_normalise to build the descriptor array. */ 1443ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lineF) { 1444ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(lineF->inUse); 1445ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_LINE_ARANGE; i++) { 1446ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[i] = lineF->w64s[i]; 1447ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1448ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_F_fetches++; 1449ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 1450ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_LINE_ARANGE; i++) { 1451ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal sv; 1452ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord ix = read_twobit_array( lineZ->ix2s, i ); 1453ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* correct, but expensive: tl_assert(ix >= 0 && ix <= 3); */ 1454ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sv = lineZ->dict[ix]; 1455ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sv != SVal_INVALID); 1456ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[i] = sv; 1457ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1458ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_Z_fetches++; 1459ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1460ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown normalise_CacheLine( cl ); 1461ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1462ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1463ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void shmem__invalidate_scache ( void ) { 1464ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word wix; 1465ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)("%s","scache inval\n"); 1466ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(!is_valid_scache_tag(1)); 1467ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (wix = 0; wix < N_WAY_NENT; wix++) { 1468ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cache_shmem.tags0[wix] = 1/*INVALID*/; 1469ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1470ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_invals++; 1471ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1472ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1473ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void shmem__flush_and_invalidate_scache ( void ) { 1474ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word wix; 1475ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr tag; 1476ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)("%s","scache flush and invalidate\n"); 1477ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(!is_valid_scache_tag(1)); 1478ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (wix = 0; wix < N_WAY_NENT; wix++) { 1479ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tag = cache_shmem.tags0[wix]; 1480ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tag == 1/*INVALID*/) { 1481ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* already invalid; nothing to do */ 1482ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 1483ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_valid_scache_tag(tag)); 1484ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cacheline_wback( wix ); 1485ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1486ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cache_shmem.tags0[wix] = 1/*INVALID*/; 1487ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1488ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_flushes++; 1489ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_invals++; 1490ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1491ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1492ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1493ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool aligned16 ( Addr a ) { 1494ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0 == (a & 1); 1495ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1496ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool aligned32 ( Addr a ) { 1497ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0 == (a & 3); 1498ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1499ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool aligned64 ( Addr a ) { 1500ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0 == (a & 7); 1501ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1502ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline UWord get_cacheline_offset ( Addr a ) { 1503ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return (UWord)(a & (N_LINE_ARANGE - 1)); 1504ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1505ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Addr cacheline_ROUNDUP ( Addr a ) { 1506ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ROUNDUP(a, N_LINE_ARANGE); 1507ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1508ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Addr cacheline_ROUNDDN ( Addr a ) { 1509ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ROUNDDN(a, N_LINE_ARANGE); 1510ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1511ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline UWord get_treeno ( Addr a ) { 1512ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return get_cacheline_offset(a) >> 3; 1513ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1514ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline UWord get_tree_offset ( Addr a ) { 1515ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return a & 7; 1516ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1517ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1518ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __attribute__((noinline)) 1519ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* get_cacheline_MISS ( Addr a ); /* fwds */ 1520ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline CacheLine* get_cacheline ( Addr a ) 1521ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1522ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* tag is 'a' with the in-line offset masked out, 1523ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown eg a[31]..a[4] 0000 */ 1524ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr tag = a & ~(N_LINE_ARANGE - 1); 1525ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord wix = (a >> N_LINE_BITS) & (N_WAY_NENT - 1); 1526ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_totrefs++; 1527ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(tag == cache_shmem.tags0[wix])) { 1528ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return &cache_shmem.lyns0[wix]; 1529ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 1530ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return get_cacheline_MISS( a ); 1531ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1532ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1533ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1534ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __attribute__((noinline)) 1535ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* get_cacheline_MISS ( Addr a ) 1536ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1537ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* tag is 'a' with the in-line offset masked out, 1538ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown eg a[31]..a[4] 0000 */ 1539ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1540ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 1541ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr* tag_old_p; 1542ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr tag = a & ~(N_LINE_ARANGE - 1); 1543ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord wix = (a >> N_LINE_BITS) & (N_WAY_NENT - 1); 1544ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1545ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(tag != cache_shmem.tags0[wix]); 1546ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1547ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Dump the old line into the backing store. */ 1548ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_totmisses++; 1549ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1550ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = &cache_shmem.lyns0[wix]; 1551ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tag_old_p = &cache_shmem.tags0[wix]; 1552ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1553ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (is_valid_scache_tag( *tag_old_p )) { 1554ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* EXPENSIVE and REDUNDANT: callee does it */ 1555ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 1556ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */ 1557ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cacheline_wback( wix ); 1558ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1559ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* and reload the new one */ 1560ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *tag_old_p = tag; 1561ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cacheline_fetch( wix ); 1562ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 1563ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */ 1564ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return cl; 1565ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1566ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1567ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UShort pulldown_to_32 ( /*MOD*/SVal* tree, UWord toff, UShort descr ) { 1568ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_64to32pulldown++; 1569ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown switch (toff) { 1570ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 0: case 4: 1571ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(descr & TREE_DESCR_64); 1572ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[4] = tree[0]; 1573ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~TREE_DESCR_64; 1574ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= (TREE_DESCR_32_1 | TREE_DESCR_32_0); 1575ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1576ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown default: 1577ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 1578ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1579ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return descr; 1580ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1581ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1582ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UShort pulldown_to_16 ( /*MOD*/SVal* tree, UWord toff, UShort descr ) { 1583ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_32to16pulldown++; 1584ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown switch (toff) { 1585ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 0: case 2: 1586ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!(descr & TREE_DESCR_32_0)) { 1587ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = pulldown_to_32(tree, 0, descr); 1588ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1589ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(descr & TREE_DESCR_32_0); 1590ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[2] = tree[0]; 1591ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~TREE_DESCR_32_0; 1592ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= (TREE_DESCR_16_1 | TREE_DESCR_16_0); 1593ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1594ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 4: case 6: 1595ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!(descr & TREE_DESCR_32_1)) { 1596ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = pulldown_to_32(tree, 4, descr); 1597ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1598ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(descr & TREE_DESCR_32_1); 1599ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[6] = tree[4]; 1600ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~TREE_DESCR_32_1; 1601ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= (TREE_DESCR_16_3 | TREE_DESCR_16_2); 1602ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1603ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown default: 1604ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 1605ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1606ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return descr; 1607ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1608ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1609ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UShort pulldown_to_8 ( /*MOD*/SVal* tree, UWord toff, UShort descr ) { 1610ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_16to8pulldown++; 1611ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown switch (toff) { 1612ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 0: case 1: 1613ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!(descr & TREE_DESCR_16_0)) { 1614ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = pulldown_to_16(tree, 0, descr); 1615ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1616ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(descr & TREE_DESCR_16_0); 1617ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[1] = tree[0]; 1618ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~TREE_DESCR_16_0; 1619ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= (TREE_DESCR_8_1 | TREE_DESCR_8_0); 1620ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1621ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 2: case 3: 1622ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!(descr & TREE_DESCR_16_1)) { 1623ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = pulldown_to_16(tree, 2, descr); 1624ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1625ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(descr & TREE_DESCR_16_1); 1626ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[3] = tree[2]; 1627ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~TREE_DESCR_16_1; 1628ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= (TREE_DESCR_8_3 | TREE_DESCR_8_2); 1629ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1630ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 4: case 5: 1631ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!(descr & TREE_DESCR_16_2)) { 1632ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = pulldown_to_16(tree, 4, descr); 1633ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1634ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(descr & TREE_DESCR_16_2); 1635ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[5] = tree[4]; 1636ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~TREE_DESCR_16_2; 1637ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= (TREE_DESCR_8_5 | TREE_DESCR_8_4); 1638ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1639ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 6: case 7: 1640ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!(descr & TREE_DESCR_16_3)) { 1641ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = pulldown_to_16(tree, 6, descr); 1642ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1643ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(descr & TREE_DESCR_16_3); 1644ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree[7] = tree[6]; 1645ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~TREE_DESCR_16_3; 1646ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= (TREE_DESCR_8_7 | TREE_DESCR_8_6); 1647ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1648ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown default: 1649ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 1650ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1651ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return descr; 1652ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1653ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1654ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1655ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UShort pullup_descr_to_16 ( UShort descr, UWord toff ) { 1656ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort mask; 1657ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown switch (toff) { 1658ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 0: 1659ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mask = TREE_DESCR_8_1 | TREE_DESCR_8_0; 1660ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert( (descr & mask) == mask ); 1661ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~mask; 1662ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= TREE_DESCR_16_0; 1663ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1664ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 2: 1665ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mask = TREE_DESCR_8_3 | TREE_DESCR_8_2; 1666ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert( (descr & mask) == mask ); 1667ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~mask; 1668ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= TREE_DESCR_16_1; 1669ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1670ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 4: 1671ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mask = TREE_DESCR_8_5 | TREE_DESCR_8_4; 1672ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert( (descr & mask) == mask ); 1673ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~mask; 1674ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= TREE_DESCR_16_2; 1675ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1676ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 6: 1677ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mask = TREE_DESCR_8_7 | TREE_DESCR_8_6; 1678ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert( (descr & mask) == mask ); 1679ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~mask; 1680ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= TREE_DESCR_16_3; 1681ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1682ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown default: 1683ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 1684ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1685ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return descr; 1686ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1687ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1688ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UShort pullup_descr_to_32 ( UShort descr, UWord toff ) { 1689ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort mask; 1690ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown switch (toff) { 1691ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 0: 1692ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!(descr & TREE_DESCR_16_0)) 1693ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = pullup_descr_to_16(descr, 0); 1694ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!(descr & TREE_DESCR_16_1)) 1695ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = pullup_descr_to_16(descr, 2); 1696ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mask = TREE_DESCR_16_1 | TREE_DESCR_16_0; 1697ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert( (descr & mask) == mask ); 1698ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~mask; 1699ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= TREE_DESCR_32_0; 1700ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1701ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 4: 1702ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!(descr & TREE_DESCR_16_2)) 1703ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = pullup_descr_to_16(descr, 4); 1704ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!(descr & TREE_DESCR_16_3)) 1705ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = pullup_descr_to_16(descr, 6); 1706ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mask = TREE_DESCR_16_3 | TREE_DESCR_16_2; 1707ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert( (descr & mask) == mask ); 1708ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr &= ~mask; 1709ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr |= TREE_DESCR_32_1; 1710ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1711ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown default: 1712ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 1713ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1714ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return descr; 1715ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1716ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1717ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Bool valid_value_is_above_me_32 ( UShort descr, UWord toff ) { 1718ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown switch (toff) { 1719ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 0: case 4: 1720ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0 != (descr & TREE_DESCR_64); 1721ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown default: 1722ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 1723ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1724ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1725ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1726ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Bool valid_value_is_below_me_16 ( UShort descr, UWord toff ) { 1727ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown switch (toff) { 1728ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 0: 1729ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0 != (descr & (TREE_DESCR_8_1 | TREE_DESCR_8_0)); 1730ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 2: 1731ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0 != (descr & (TREE_DESCR_8_3 | TREE_DESCR_8_2)); 1732ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 4: 1733ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0 != (descr & (TREE_DESCR_8_5 | TREE_DESCR_8_4)); 1734ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown case 6: 1735ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0 != (descr & (TREE_DESCR_8_7 | TREE_DESCR_8_6)); 1736ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown default: 1737ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 1738ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1739ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1740ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1741ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* ------------ Cache management ------------ */ 1742ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1743ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_flush_cache ( void ) 1744ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1745ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown shmem__flush_and_invalidate_scache(); 1746ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1747ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1748ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1749ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_init ( void(*p_rcinc)(SVal), void(*p_rcdec)(SVal) ) 1750ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1751ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert( sizeof(UWord) == sizeof(Addr) ); 1752ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1753ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcinc = p_rcinc; 1754ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcdec = p_rcdec; 1755ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1756ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(map_shmem == NULL); 1757ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown map_shmem = VG_(newFM)( HG_(zalloc), "libhb.zsm_init.1 (map_shmem)", 1758ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free), 1759ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown NULL/*unboxed UWord cmp*/); 1760ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(map_shmem != NULL); 1761ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown shmem__invalidate_scache(); 1762ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1763ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* a SecMap must contain an integral number of CacheLines */ 1764ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0 == (N_SECMAP_ARANGE % N_LINE_ARANGE)); 1765ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* also ... a CacheLine holds an integral number of trees */ 1766ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0 == (N_LINE_ARANGE % 8)); 1767ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1768ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1769ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 1770ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 1771ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 1772ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// SECTION END compressed shadow memory // 1773ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 1774ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 1775ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 1776ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1777ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1778ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1779ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 1780ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 1781ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 1782ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// SECTION BEGIN vts primitives // 1783ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 1784ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 1785ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 1786ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1787ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1788b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* There's a 1-1 mapping between Thr and ThrIDs -- the latter merely 1789b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov being compact stand-ins for Thr*'s. Use these functions to map 1790b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov between them. */ 1791b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic ThrID Thr__to_ThrID ( Thr* thr ); /* fwds */ 1792b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic Thr* Thr__from_ThrID ( ThrID thrid ); /* fwds */ 1793b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1794b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov__attribute__((noreturn)) 1795b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void scalarts_limitations_fail_NORETURN ( Bool due_to_nThrs ) 1796b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 1797b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (due_to_nThrs) { 1798b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov HChar* s = 1799b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "\n" 1800b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "Helgrind: cannot continue, run aborted: too many threads.\n" 1801b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "Sorry. Helgrind can only handle programs that create\n" 1802b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "%'llu or fewer threads over their entire lifetime.\n" 1803b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "\n"; 1804b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(umsg)(s, (ULong)(ThrID_MAX_VALID - 1024)); 1805b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } else { 1806b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov HChar* s = 1807b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "\n" 1808b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "Helgrind: cannot continue, run aborted: too many\n" 1809b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "synchronisation events. Sorry. Helgrind can only handle\n" 1810b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "programs which perform %'llu or fewer\n" 1811b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "inter-thread synchronisation events (locks, unlocks, etc).\n" 1812b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "\n"; 1813b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(umsg)(s, (1ULL << SCALARTS_N_TYMBITS) - 1); 1814b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 1815b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(exit)(1); 1816b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /*NOTREACHED*/ 1817b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(0); /*wtf?!*/ 1818b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 1819b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1820b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1821b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* The dead thread (ThrID, actually) table. A thread may only be 1822b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov listed here if we have been notified thereof by libhb_async_exit. 1823b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov New entries are added at the end. The order isn't important, but 1824b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov the ThrID values must be unique. This table lists the identity of 1825b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov all threads that have ever died -- none are ever removed. We keep 1826b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov this table so as to be able to prune entries from VTSs. We don't 1827b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov actually need to keep the set of threads that have ever died -- 1828b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov only the threads that have died since the previous round of 1829b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov pruning. But it's useful for sanity check purposes to keep the 1830b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov entire set, so we do. */ 1831b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic XArray* /* of ThrID */ verydead_thread_table = NULL; 1832b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1833b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Arbitrary total ordering on ThrIDs. */ 1834b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic Int cmp__ThrID ( void* v1, void* v2 ) { 1835b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID id1 = *(ThrID*)v1; 1836b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID id2 = *(ThrID*)v2; 1837b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (id1 < id2) return -1; 1838b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (id1 > id2) return 1; 1839b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return 0; 1840b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 1841b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1842b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void verydead_thread_table_init ( void ) 1843b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 1844b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(!verydead_thread_table); 1845b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov verydead_thread_table 1846b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov = VG_(newXA)( HG_(zalloc), 1847b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "libhb.verydead_thread_table_init.1", 1848b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov HG_(free), sizeof(ThrID) ); 1849b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(verydead_thread_table); 1850b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(setCmpFnXA)(verydead_thread_table, cmp__ThrID); 1851b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 1852b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1853ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1854ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* A VTS contains .ts, its vector clock, and also .id, a field to hold 1855ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a backlink for the caller's convenience. Since we have no idea 1856ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown what to set that to in the library, it always gets set to 1857ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID_INVALID. */ 1858ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef 1859ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct { 1860b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID id; 1861b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt usedTS; 1862b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt sizeTS; 1863b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS ts[0]; 1864ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1865ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VTS; 1866ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1867b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Allocate a VTS capable of storing 'sizeTS' entries. */ 1868b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic VTS* VTS__new ( HChar* who, UInt sizeTS ); 1869b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1870b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Make a clone of 'vts', sizing the new array to exactly match the 1871b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov number of ScalarTSs present. */ 1872b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic VTS* VTS__clone ( HChar* who, VTS* vts ); 1873ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1874b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Make a clone of 'vts' with the thrids in 'thrids' removed. The new 1875b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov array is sized exactly to hold the number of required elements. 1876b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 'thridsToDel' is an array of ThrIDs to be omitted in the clone, and 1877b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov must be in strictly increasing order. */ 1878b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic VTS* VTS__subtract ( HChar* who, VTS* vts, XArray* thridsToDel ); 1879ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1880ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Delete this VTS in its entirety. */ 1881ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void VTS__delete ( VTS* vts ); 1882ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1883b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Create a new singleton VTS in 'out'. Caller must have 1884b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov pre-allocated 'out' sufficiently big to hold the result in all 1885b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov possible cases. */ 1886b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void VTS__singleton ( /*OUT*/VTS* out, Thr* thr, ULong tym ); 1887ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1888b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Create in 'out' a VTS which is the same as 'vts' except with 1889b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov vts[me]++, so to speak. Caller must have pre-allocated 'out' 1890b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov sufficiently big to hold the result in all possible cases. */ 1891b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void VTS__tick ( /*OUT*/VTS* out, Thr* me, VTS* vts ); 1892ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1893b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Create in 'out' a VTS which is the join (max) of 'a' and 1894b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 'b'. Caller must have pre-allocated 'out' sufficiently big to hold 1895b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov the result in all possible cases. */ 1896b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void VTS__join ( /*OUT*/VTS* out, VTS* a, VTS* b ); 1897ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1898ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Compute the partial ordering relation of the two args. Although we 1899ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown could be completely general and return an enumeration value (EQ, 1900ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LT, GT, UN), in fact we only need LEQ, and so we may as well 1901ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown hardwire that fact. 1902ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1903b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Returns zero iff LEQ(A,B), or a valid ThrID if not (zero is an 1904b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov invald ThrID). In the latter case, the returned ThrID indicates 1905b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov the discovered point for which they are not. There may be more 1906b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov than one such point, but we only care about seeing one of them, not 1907b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov all of them. This rather strange convention is used because 1908b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov sometimes we want to know the actual index at which they first 1909b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov differ. */ 1910b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic UInt VTS__cmpLEQ ( VTS* a, VTS* b ); 1911ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1912ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Compute an arbitrary structural (total) ordering on the two args, 1913ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown based on their VCs, so they can be looked up in a table, tree, etc. 1914ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Returns -1, 0 or 1. */ 1915ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Word VTS__cmp_structural ( VTS* a, VTS* b ); 1916ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1917ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Debugging only. Display the given VTS in the buffer. */ 1918ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void VTS__show ( HChar* buf, Int nBuf, VTS* vts ); 1919ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1920ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Debugging only. Return vts[index], so to speak. */ 1921ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic ULong VTS__indexAt_SLOW ( VTS* vts, Thr* idx ); 1922ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1923b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Notify the VTS machinery that a thread has been declared 1924b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov comprehensively dead: that is, it has done an async exit AND it has 1925b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov been joined with. This should ensure that its local clocks (.viR 1926b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov and .viW) will never again change, and so all mentions of this 1927b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thread from all VTSs in the system may be removed. */ 1928b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void VTS__declare_thread_very_dead ( Thr* idx ); 1929ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1930ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--------------- to do with Vector Timestamps ---------------*/ 1931ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1932ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Bool is_sane_VTS ( VTS* vts ) 1933ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1934ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i, n; 1935ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ScalarTS *st1, *st2; 1936ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!vts) return False; 1937ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!vts->ts) return False; 1938b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (vts->usedTS > vts->sizeTS) return False; 1939b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov n = vts->usedTS; 1940b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (n == 1) { 1941b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov st1 = &vts->ts[0]; 1942b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (st1->tym == 0) 1943b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return False; 1944b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 1945b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov else 1946ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n >= 2) { 1947ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < n-1; i++) { 1948b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov st1 = &vts->ts[i]; 1949b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov st2 = &vts->ts[i+1]; 1950b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (st1->thrid >= st2->thrid) 1951ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 1952ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (st1->tym == 0 || st2->tym == 0) 1953ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 1954ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1955ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1956ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return True; 1957ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1958ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1959ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1960ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Create a new, empty VTS. 1961ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 1962b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic VTS* VTS__new ( HChar* who, UInt sizeTS ) 19638f943afc22a6a683b78271836c8ddc462b4824a9Evgeniy Stepanov{ 1964b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS* vts = HG_(zalloc)(who, sizeof(VTS) + (sizeTS+1) * sizeof(ScalarTS)); 1965b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(vts->usedTS == 0); 1966b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov vts->sizeTS = sizeTS; 1967b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov *(ULong*)(&vts->ts[sizeTS]) = 0x0ddC0ffeeBadF00dULL; 19689bea4c13fca0e3bb4b719dcb3ed63d47d479294eKenny Root return vts; 19698f943afc22a6a683b78271836c8ddc462b4824a9Evgeniy Stepanov} 19708f943afc22a6a683b78271836c8ddc462b4824a9Evgeniy Stepanov 1971b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Clone this VTS. 1972b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov*/ 1973b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic VTS* VTS__clone ( HChar* who, VTS* vts ) 1974b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 1975b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(vts); 1976b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert( *(ULong*)(&vts->ts[vts->sizeTS]) == 0x0ddC0ffeeBadF00dULL); 1977b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt nTS = vts->usedTS; 1978b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS* clone = VTS__new(who, nTS); 1979b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov clone->id = vts->id; 1980b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov clone->sizeTS = nTS; 1981b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov clone->usedTS = nTS; 1982b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt i; 1983b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for (i = 0; i < nTS; i++) { 1984b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov clone->ts[i] = vts->ts[i]; 1985b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 1986b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert( *(ULong*)(&clone->ts[clone->sizeTS]) == 0x0ddC0ffeeBadF00dULL); 1987b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return clone; 1988b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 1989b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1990b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1991b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Make a clone of a VTS with specified ThrIDs removed. 'thridsToDel' 1992b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov must be in strictly increasing order. We could obviously do this 1993b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov much more efficiently (in linear time) if necessary. 1994b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov*/ 1995b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic VTS* VTS__subtract ( HChar* who, VTS* vts, XArray* thridsToDel ) 1996b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 1997b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt i, j; 1998b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(vts); 1999b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(thridsToDel); 2000b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert( *(ULong*)(&vts->ts[vts->sizeTS]) == 0x0ddC0ffeeBadF00dULL); 2001b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt nTS = vts->usedTS; 2002b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Figure out how many ScalarTSs will remain in the output. */ 2003b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt nReq = nTS; 2004b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for (i = 0; i < nTS; i++) { 2005b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID thrid = vts->ts[i].thrid; 2006b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (VG_(lookupXA)(thridsToDel, &thrid, NULL, NULL)) 2007b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov nReq--; 2008b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 2009b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(nReq <= nTS); 2010b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Copy the ones that will remain. */ 2011b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS* res = VTS__new(who, nReq); 2012b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov j = 0; 2013b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for (i = 0; i < nTS; i++) { 2014b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID thrid = vts->ts[i].thrid; 2015b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (VG_(lookupXA)(thridsToDel, &thrid, NULL, NULL)) 2016b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov continue; 2017b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov res->ts[j++] = vts->ts[i]; 2018b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 2019b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(j == nReq); 2020b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(j == res->sizeTS); 2021b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov res->usedTS = j; 2022b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert( *(ULong*)(&res->ts[j]) == 0x0ddC0ffeeBadF00dULL); 2023b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return res; 2024b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 2025b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2026ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2027ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Delete this VTS in its entirety. 2028ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 2029b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void VTS__delete ( VTS* vts ) 2030ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2031ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(vts); 2032b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(vts->usedTS <= vts->sizeTS); 2033b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert( *(ULong*)(&vts->ts[vts->sizeTS]) == 0x0ddC0ffeeBadF00dULL); 2034ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free)(vts); 2035ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2036ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2037ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2038ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Create a new singleton VTS. 2039ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 2040b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void VTS__singleton ( /*OUT*/VTS* out, Thr* thr, ULong tym ) 2041b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 2042ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(thr); 2043ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(tym >= 1); 2044b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(out); 2045b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(out->usedTS == 0); 2046b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(out->sizeTS >= 1); 2047b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt hi = out->usedTS++; 2048b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov out->ts[hi].thrid = Thr__to_ThrID(thr); 2049b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov out->ts[hi].tym = tym; 2050ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2051ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2052ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2053ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Return a new VTS in which vts[me]++, so to speak. 'vts' itself is 2054ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown not modified. 2055ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 2056b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void VTS__tick ( /*OUT*/VTS* out, Thr* me, VTS* vts ) 2057ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2058b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt i, n; 2059b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID me_thrid; 2060b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Bool found = False; 2061ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2062ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__vts__tick++; 2063ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2064b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(out); 2065b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(out->usedTS == 0); 2066b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (vts->usedTS >= ThrID_MAX_VALID) 2067b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov scalarts_limitations_fail_NORETURN( True/*due_to_nThrs*/ ); 2068b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(out->sizeTS >= 1 + vts->usedTS); 2069b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2070ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(me); 2071b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov me_thrid = Thr__to_ThrID(me); 2072ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_VTS(vts)); 2073b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov n = vts->usedTS; 2074ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2075b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Copy all entries which precede 'me'. */ 2076ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < n; i++) { 2077b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS* here = &vts->ts[i]; 2078b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (UNLIKELY(here->thrid >= me_thrid)) 2079ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 2080b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt hi = out->usedTS++; 2081b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov out->ts[hi] = *here; 2082b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 2083b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2084b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* 'i' now indicates the next entry to copy, if any. 2085b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov There are 3 possibilities: 2086b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov (a) there is no next entry (we used them all up already): 2087b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov add (me_thrid,1) to the output, and quit 2088b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov (b) there is a next entry, and its thrid > me_thrid: 2089b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov add (me_thrid,1) to the output, then copy the remaining entries 2090b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov (c) there is a next entry, and its thrid == me_thrid: 2091b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov copy it to the output but increment its timestamp value. 2092b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Then copy the remaining entries. (c) is the common case. 2093b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov */ 20949bea4c13fca0e3bb4b719dcb3ed63d47d479294eKenny Root tl_assert(i >= 0 && i <= n); 2095b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (i == n) { /* case (a) */ 2096b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt hi = out->usedTS++; 2097b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov out->ts[hi].thrid = me_thrid; 2098b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov out->ts[hi].tym = 1; 20999bea4c13fca0e3bb4b719dcb3ed63d47d479294eKenny Root } else { 2100b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* cases (b) and (c) */ 2101b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS* here = &vts->ts[i]; 2102b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (me_thrid == here->thrid) { /* case (c) */ 2103b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (UNLIKELY(here->tym >= (1ULL << SCALARTS_N_TYMBITS) - 2ULL)) { 2104b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* We're hosed. We have to stop. */ 2105b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov scalarts_limitations_fail_NORETURN( False/*!due_to_nThrs*/ ); 2106b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 2107b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt hi = out->usedTS++; 2108b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov out->ts[hi].thrid = here->thrid; 2109b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov out->ts[hi].tym = here->tym + 1; 2110b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov i++; 2111b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov found = True; 2112b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } else { /* case (b) */ 2113b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt hi = out->usedTS++; 2114b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov out->ts[hi].thrid = me_thrid; 2115b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov out->ts[hi].tym = 1; 2116b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 2117b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* And copy any remaining entries. */ 2118ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (/*keepgoing*/; i < n; i++) { 2119b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS* here2 = &vts->ts[i]; 2120b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt hi = out->usedTS++; 2121b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov out->ts[hi] = *here2; 2122ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2123ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2124b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2125b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(is_sane_VTS(out)); 2126b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(out->usedTS == vts->usedTS + (found ? 0 : 1)); 2127b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(out->usedTS <= out->sizeTS); 2128ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2129ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2130ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2131ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Return a new VTS constructed as the join (max) of the 2 args. 2132ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Neither arg is modified. 2133ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 2134b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void VTS__join ( /*OUT*/VTS* out, VTS* a, VTS* b ) 2135ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2136b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt ia, ib, useda, usedb; 2137ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ULong tyma, tymb, tymMax; 2138b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID thrid; 2139b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt ncommon = 0; 2140ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2141ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__vts__join++; 2142ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2143b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(a); 2144b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(b); 2145b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov useda = a->usedTS; 2146b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov usedb = b->usedTS; 2147b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2148b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(out); 2149b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(out->usedTS == 0); 2150b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* overly conservative test, but doing better involves comparing 2151b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov the two VTSs, which we don't want to do at this point. */ 2152b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (useda + usedb >= ThrID_MAX_VALID) 2153b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov scalarts_limitations_fail_NORETURN( True/*due_to_nThrs*/ ); 2154b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(out->sizeTS >= useda + usedb); 2155ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2156ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ia = ib = 0; 2157ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2158ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (1) { 2159ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2160b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* This logic is to enumerate triples (thrid, tyma, tymb) drawn 2161b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov from a and b in order, where thrid is the next ThrID 2162ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown occurring in either a or b, and tyma/b are the relevant 2163ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown scalar timestamps, taking into account implicit zeroes. */ 2164ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ia >= 0 && ia <= useda); 2165ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ib >= 0 && ib <= usedb); 2166ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2167ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (ia == useda && ib == usedb) { 2168ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* both empty - done */ 2169ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 2170ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2171ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else if (ia == useda && ib != usedb) { 2172ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* a empty, use up b */ 2173b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS* tmpb = &b->ts[ib]; 2174b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid = tmpb->thrid; 2175b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tyma = 0; 2176b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tymb = tmpb->tym; 2177ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ib++; 2178ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2179ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else if (ia != useda && ib == usedb) { 2180ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* b empty, use up a */ 2181b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS* tmpa = &a->ts[ia]; 2182b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid = tmpa->thrid; 2183b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tyma = tmpa->tym; 2184b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tymb = 0; 2185ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ia++; 2186ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2187ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 2188b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* both not empty; extract lowest-ThrID'd triple */ 2189b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS* tmpa = &a->ts[ia]; 2190b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS* tmpb = &b->ts[ib]; 2191b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (tmpa->thrid < tmpb->thrid) { 2192b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* a has the lowest unconsidered ThrID */ 2193b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid = tmpa->thrid; 2194b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tyma = tmpa->tym; 2195b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tymb = 0; 2196ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ia++; 2197b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } else if (tmpa->thrid > tmpb->thrid) { 2198b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* b has the lowest unconsidered ThrID */ 2199b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid = tmpb->thrid; 2200b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tyma = 0; 2201b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tymb = tmpb->tym; 2202ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ib++; 2203ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 2204b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* they both next mention the same ThrID */ 2205b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(tmpa->thrid == tmpb->thrid); 2206b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid = tmpa->thrid; /* == tmpb->thrid */ 2207b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tyma = tmpa->tym; 2208b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tymb = tmpb->tym; 2209ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ia++; 2210ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ib++; 2211b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ncommon++; 2212ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2213ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2214ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2215ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* having laboriously determined (thr, tyma, tymb), do something 2216ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown useful with it. */ 2217ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tymMax = tyma > tymb ? tyma : tymb; 2218ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tymMax > 0) { 2219b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt hi = out->usedTS++; 2220b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov out->ts[hi].thrid = thrid; 2221b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov out->ts[hi].tym = tymMax; 2222ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2223ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2224ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2225ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2226b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(is_sane_VTS(out)); 2227b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(out->usedTS <= out->sizeTS); 2228b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(out->usedTS == useda + usedb - ncommon); 2229ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2230ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2231ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2232b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Determine if 'a' <= 'b', in the partial ordering. Returns zero if 2233b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov they are, or the first ThrID for which they are not (no valid ThrID 2234b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov has the value zero). This rather strange convention is used 2235b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov because sometimes we want to know the actual index at which they 2236b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov first differ. */ 2237b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic UInt/*ThrID*/ VTS__cmpLEQ ( VTS* a, VTS* b ) 2238ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2239ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word ia, ib, useda, usedb; 2240ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ULong tyma, tymb; 2241ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2242ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__vts__cmpLEQ++; 2243ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2244b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(a); 2245b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(b); 2246b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov useda = a->usedTS; 2247b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov usedb = b->usedTS; 2248ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2249ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ia = ib = 0; 2250ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2251ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (1) { 2252ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2253ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* This logic is to enumerate doubles (tyma, tymb) drawn 2254ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown from a and b in order, and tyma/b are the relevant 2255ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown scalar timestamps, taking into account implicit zeroes. */ 2256b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID thrid; 2257ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2258ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ia >= 0 && ia <= useda); 2259ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ib >= 0 && ib <= usedb); 2260ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2261ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (ia == useda && ib == usedb) { 2262ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* both empty - done */ 2263ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 2264ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2265ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else if (ia == useda && ib != usedb) { 2266ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* a empty, use up b */ 2267b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS* tmpb = &b->ts[ib]; 2268b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tyma = 0; 2269b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tymb = tmpb->tym; 2270b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid = tmpb->thrid; 2271ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ib++; 2272ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2273ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else if (ia != useda && ib == usedb) { 2274ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* b empty, use up a */ 2275b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS* tmpa = &a->ts[ia]; 2276b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tyma = tmpa->tym; 2277b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid = tmpa->thrid; 2278b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tymb = 0; 2279ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ia++; 2280ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2281ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 2282b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* both not empty; extract lowest-ThrID'd triple */ 2283b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS* tmpa = &a->ts[ia]; 2284b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS* tmpb = &b->ts[ib]; 2285b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (tmpa->thrid < tmpb->thrid) { 2286b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* a has the lowest unconsidered ThrID */ 2287b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tyma = tmpa->tym; 2288b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid = tmpa->thrid; 2289b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tymb = 0; 2290ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ia++; 2291ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2292ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown else 2293b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (tmpa->thrid > tmpb->thrid) { 2294b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* b has the lowest unconsidered ThrID */ 2295b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tyma = 0; 2296b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tymb = tmpb->tym; 2297b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid = tmpb->thrid; 2298ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ib++; 2299ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 2300b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* they both next mention the same ThrID */ 2301b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(tmpa->thrid == tmpb->thrid); 2302b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tyma = tmpa->tym; 2303b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid = tmpa->thrid; 2304b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tymb = tmpb->tym; 2305ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ia++; 2306ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ib++; 2307ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2308ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2309ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2310ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* having laboriously determined (tyma, tymb), do something 2311ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown useful with it. */ 2312ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tyma > tymb) { 2313ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* not LEQ at this index. Quit, since the answer is 2314ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown determined already. */ 2315b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(thrid >= 1024); 2316b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return thrid; 2317ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2318ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2319ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2320b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return 0; /* all points are LEQ => return an invalid ThrID */ 2321ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2322ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2323ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2324ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Compute an arbitrary structural (total) ordering on the two args, 2325ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown based on their VCs, so they can be looked up in a table, tree, etc. 2326ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Returns -1, 0 or 1. (really just 'deriving Ord' :-) This can be 2327ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown performance critical so there is some effort expended to make it sa 2328ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fast as possible. 2329ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 2330ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownWord VTS__cmp_structural ( VTS* a, VTS* b ) 2331ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2332ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* We just need to generate an arbitrary total ordering based on 2333ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a->ts and b->ts. Preferably do it in a way which comes across likely 2334ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown differences relatively quickly. */ 2335ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i; 2336ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word useda = 0, usedb = 0; 2337ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ScalarTS *ctsa = NULL, *ctsb = NULL; 2338ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2339ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__vts__cmp_structural++; 2340ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2341ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(a); 2342ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(b); 2343ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2344b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ctsa = &a->ts[0]; useda = a->usedTS; 2345b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ctsb = &b->ts[0]; usedb = b->usedTS; 2346ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2347ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(useda == usedb)) { 2348ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ScalarTS *tmpa = NULL, *tmpb = NULL; 2349ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__vts__cmp_structural_slow++; 2350ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Same length vectors. Find the first difference, if any, as 2351ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fast as possible. */ 2352ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < useda; i++) { 2353ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tmpa = &ctsa[i]; 2354ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tmpb = &ctsb[i]; 2355b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (LIKELY(tmpa->tym == tmpb->tym 2356b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov && tmpa->thrid == tmpb->thrid)) 2357ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown continue; 2358ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown else 2359ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 2360ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2361ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(i == useda)) { 2362ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* They're identical. */ 2363ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0; 2364ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 2365ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(i >= 0 && i < useda); 2366ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tmpa->tym < tmpb->tym) return -1; 2367ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tmpa->tym > tmpb->tym) return 1; 2368b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (tmpa->thrid < tmpb->thrid) return -1; 2369b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (tmpa->thrid > tmpb->thrid) return 1; 2370ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* we just established them as non-identical, hence: */ 2371ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2372ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /*NOTREACHED*/ 2373ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 2374ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2375ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2376ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (useda < usedb) return -1; 2377ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (useda > usedb) return 1; 2378ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /*NOTREACHED*/ 2379ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 2380ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2381ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2382ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2383ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Debugging only. Display the given VTS in the buffer. 2384ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 2385b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovvoid VTS__show ( HChar* buf, Int nBuf, VTS* vts ) 2386b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 2387ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ScalarTS* st; 2388ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HChar unit[64]; 2389ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i, n; 2390ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Int avail = nBuf; 2391ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(vts && vts->ts); 2392ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(nBuf > 16); 2393ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown buf[0] = '['; 2394ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown buf[1] = 0; 2395b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov n = vts->usedTS; 2396ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < n; i++) { 2397ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(avail >= 40); 2398b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov st = &vts->ts[i]; 2399ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(memset)(unit, 0, sizeof(unit)); 2400b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(sprintf)(unit, i < n-1 ? "%u:%llu " : "%u:%llu", 2401b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov st->thrid, (ULong)st->tym); 2402ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (avail < VG_(strlen)(unit) + 40/*let's say*/) { 2403ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(strcat)(buf, " ...]"); 2404ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown buf[nBuf-1] = 0; 2405ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 2406ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2407ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(strcat)(buf, unit); 2408ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown avail -= VG_(strlen)(unit); 2409ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2410ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(strcat)(buf, "]"); 2411ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown buf[nBuf-1] = 0; 2412ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2413ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2414ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2415ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Debugging only. Return vts[index], so to speak. 2416ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 2417b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy StepanovULong VTS__indexAt_SLOW ( VTS* vts, Thr* idx ) 2418b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 2419ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i, n; 2420b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID idx_thrid = Thr__to_ThrID(idx); 2421ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__vts__indexat_slow++; 2422ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(vts && vts->ts); 2423b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov n = vts->usedTS; 2424ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < n; i++) { 2425b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ScalarTS* st = &vts->ts[i]; 2426b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (st->thrid == idx_thrid) 2427ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return st->tym; 2428ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2429ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0; 2430ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2431ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2432ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2433b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* See comment on prototype above. 2434b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov*/ 2435b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void VTS__declare_thread_very_dead ( Thr* thr ) 2436b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 2437b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (0) VG_(printf)("VTQ: tae %p\n", thr); 2438b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2439b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(thr->llexit_done); 2440b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(thr->joinedwith_done); 2441b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2442b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID nyu; 2443b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov nyu = Thr__to_ThrID(thr); 2444b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(addToXA)( verydead_thread_table, &nyu ); 2445b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2446b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* We can only get here if we're assured that we'll never again 2447b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov need to look at this thread's ::viR or ::viW. Set them to 2448b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID_INVALID, partly so as to avoid holding on to the VTSs, but 2449b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov mostly so that we don't wind up pruning them (as that would be 2450b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov nonsensical: the only interesting ScalarTS entry for a dead 2451b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thread is its own index, and the pruning will remove that.). */ 2452b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID__rcdec(thr->viR); 2453b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID__rcdec(thr->viW); 2454b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thr->viR = VtsID_INVALID; 2455b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thr->viW = VtsID_INVALID; 2456b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 2457b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2458b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2459ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 2460ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 2461ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 2462ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// SECTION END vts primitives // 2463ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 2464ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 2465ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 2466ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2467ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2468ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2469ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 2470ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 2471ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 2472ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// SECTION BEGIN main library // 2473ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 2474ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 2475ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 2476ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2477ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2478ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 2479ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 2480ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// VTS set // 2481ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 2482ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 2483ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2484b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic WordFM* /* WordFM VTS* void */ vts_set = NULL; 2485ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2486ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void vts_set_init ( void ) 2487ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2488ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(!vts_set); 2489ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts_set = VG_(newFM)( HG_(zalloc), "libhb.vts_set_init.1", 2490ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free), 2491ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Word(*)(UWord,UWord))VTS__cmp_structural ); 2492ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(vts_set); 2493ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2494ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2495b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Given a VTS, look in vts_set to see if we already have a 2496b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov structurally identical one. If yes, return the pair (True, pointer 2497b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov to the existing one). If no, clone this one, add the clone to the 2498b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov set, and return (False, pointer to the clone). */ 2499b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic Bool vts_set__find__or__clone_and_add ( /*OUT*/VTS** res, VTS* cand ) 2500ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2501ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord keyW, valW; 2502b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov stats__vts_set__focaa++; 2503b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(cand->id == VtsID_INVALID); 2504ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* lookup cand (by value) */ 2505ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (VG_(lookupFM)( vts_set, &keyW, &valW, (UWord)cand )) { 2506ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* found it */ 2507ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(valW == 0); 2508ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* if this fails, cand (by ref) was already present (!) */ 2509ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(keyW != (UWord)cand); 2510b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov *res = (VTS*)keyW; 2511b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return True; 2512ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 2513b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* not present. Clone, add and return address of clone. */ 2514b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov stats__vts_set__focaa_a++; 2515b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS* clone = VTS__clone( "libhb.vts_set_focaa.1", cand ); 2516b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(clone != cand); 2517b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(addToFM)( vts_set, (UWord)clone, 0/*val is unused*/ ); 2518b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov *res = clone; 2519b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return False; 2520ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2521ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2522ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2523ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2524ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 2525ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 2526ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// VTS table // 2527ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 2528ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 2529ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2530ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void VtsID__invalidate_caches ( void ); /* fwds */ 2531ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2532ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* A type to hold VTS table entries. Invariants: 2533ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown If .vts == NULL, then this entry is not in use, so: 2534ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown - .rc == 0 2535ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown - this entry is on the freelist (unfortunately, does not imply 2536b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov any constraints on value for .freelink) 2537ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown If .vts != NULL, then this entry is in use: 2538ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown - .vts is findable in vts_set 2539ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown - .vts->id == this entry number 2540ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown - no specific value for .rc (even 0 is OK) 2541b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov - this entry is not on freelist, so .freelink == VtsID_INVALID 2542ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 2543ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef 2544ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct { 2545ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VTS* vts; /* vts, in vts_set */ 2546ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord rc; /* reference count - enough for entire aspace */ 2547ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID freelink; /* chain for free entries, VtsID_INVALID at end */ 2548b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID remap; /* used only during pruning */ 2549ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2550ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsTE; 2551ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2552ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* The VTS table. */ 2553ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic XArray* /* of VtsTE */ vts_tab = NULL; 2554ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2555ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* An index into the VTS table, indicating the start of the list of 2556ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown free (available for use) entries. If the list is empty, this is 2557ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID_INVALID. */ 2558ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic VtsID vts_tab_freelist = VtsID_INVALID; 2559ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2560ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Do a GC of vts_tab when the freelist becomes empty AND the size of 2561ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts_tab equals or exceeds this size. After GC, the value here is 2562ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown set appropriately so as to check for the next GC point. */ 2563ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Word vts_next_GC_at = 1000; 2564ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2565ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void vts_tab_init ( void ) 2566ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2567ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts_tab 2568ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown = VG_(newXA)( HG_(zalloc), "libhb.vts_tab_init.1", 2569ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free), sizeof(VtsTE) ); 2570ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts_tab_freelist 2571ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown = VtsID_INVALID; 2572ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(vts_tab); 2573ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2574ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2575ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Add ii to the free list, checking that it looks out-of-use. */ 2576ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void add_to_free_list ( VtsID ii ) 2577ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2578ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsTE* ie = VG_(indexXA)( vts_tab, ii ); 2579ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ie->vts == NULL); 2580ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ie->rc == 0); 2581ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ie->freelink == VtsID_INVALID); 2582ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ie->freelink = vts_tab_freelist; 2583ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts_tab_freelist = ii; 2584ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2585ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2586ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Get an entry from the free list. This will return VtsID_INVALID if 2587ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown the free list is empty. */ 2588ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic VtsID get_from_free_list ( void ) 2589ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2590ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID ii; 2591ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsTE* ie; 2592ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (vts_tab_freelist == VtsID_INVALID) 2593ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return VtsID_INVALID; 2594ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ii = vts_tab_freelist; 2595ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ie = VG_(indexXA)( vts_tab, ii ); 2596ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ie->vts == NULL); 2597ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ie->rc == 0); 2598ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts_tab_freelist = ie->freelink; 2599ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ii; 2600ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2601ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2602ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Produce a new VtsID that can be used, either by getting it from 2603ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown the freelist, or, if that is empty, by expanding vts_tab. */ 2604ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic VtsID get_new_VtsID ( void ) 2605ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2606ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID ii; 2607ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsTE te; 2608ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ii = get_from_free_list(); 2609ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (ii != VtsID_INVALID) 2610ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ii; 2611ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown te.vts = NULL; 2612ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown te.rc = 0; 2613ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown te.freelink = VtsID_INVALID; 2614b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov te.remap = VtsID_INVALID; 2615ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ii = (VtsID)VG_(addToXA)( vts_tab, &te ); 2616ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ii; 2617ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2618ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2619ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2620ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Indirect callback from lib_zsm. */ 2621ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void VtsID__rcinc ( VtsID ii ) 2622ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2623ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsTE* ie; 2624ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* VG_(indexXA) does a range check for us */ 2625ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ie = VG_(indexXA)( vts_tab, ii ); 2626ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ie->vts); /* else it's not in use */ 2627ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ie->rc < ~0UL); /* else we can't continue */ 2628ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ie->vts->id == ii); 2629ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ie->rc++; 2630ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2631ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2632ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Indirect callback from lib_zsm. */ 2633ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void VtsID__rcdec ( VtsID ii ) 2634ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2635ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsTE* ie; 2636ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* VG_(indexXA) does a range check for us */ 2637ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ie = VG_(indexXA)( vts_tab, ii ); 2638ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ie->vts); /* else it's not in use */ 2639ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ie->rc > 0); /* else RC snafu */ 2640ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ie->vts->id == ii); 2641ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ie->rc--; 2642ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2643ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2644ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2645b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Look up 'cand' in our collection of VTSs. If present, return the 2646b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID for the pre-existing version. If not present, clone it, add 2647b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov the clone to both vts_tab and vts_set, allocate a fresh VtsID for 2648b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov it, and return that. */ 2649b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic VtsID vts_tab__find__or__clone_and_add ( VTS* cand ) 2650ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2651b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS* in_tab = NULL; 2652ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(cand->id == VtsID_INVALID); 2653b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Bool already_have = vts_set__find__or__clone_and_add( &in_tab, cand ); 2654b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(in_tab); 2655b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (already_have) { 2656b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* We already have a copy of 'cand'. Use that. */ 2657ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsTE* ie; 2658b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(in_tab->id != VtsID_INVALID); 2659b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ie = VG_(indexXA)( vts_tab, in_tab->id ); 2660b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(ie->vts == in_tab); 2661b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return in_tab->id; 2662ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 2663ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID ii = get_new_VtsID(); 2664ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsTE* ie = VG_(indexXA)( vts_tab, ii ); 2665b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ie->vts = in_tab; 2666ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ie->rc = 0; 2667ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ie->freelink = VtsID_INVALID; 2668b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov in_tab->id = ii; 2669ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ii; 2670ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2671ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2672ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2673ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2674ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void show_vts_stats ( HChar* caller ) 2675ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2676ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord nSet, nTab, nLive; 2677ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ULong totrc; 2678ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord n, i; 2679ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nSet = VG_(sizeFM)( vts_set ); 2680ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nTab = VG_(sizeXA)( vts_tab ); 2681ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown totrc = 0; 2682ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nLive = 0; 2683ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown n = VG_(sizeXA)( vts_tab ); 2684ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < n; i++) { 2685ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsTE* ie = VG_(indexXA)( vts_tab, i ); 2686ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (ie->vts) { 2687ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nLive++; 2688ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown totrc += (ULong)ie->rc; 2689ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 2690ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ie->rc == 0); 2691ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2692ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2693ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" show_vts_stats %s\n", caller); 2694ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" vts_tab size %4lu\n", nTab); 2695ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" vts_tab live %4lu\n", nLive); 2696ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" vts_set size %4lu\n", nSet); 2697ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" total rc %4llu\n", totrc); 2698ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2699ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2700b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2701b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* --- Helpers for VtsID pruning --- */ 2702b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2703b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic 2704b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovvoid remap_VtsID ( /*MOD*/XArray* /* of VtsTE */ old_tab, 2705b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /*MOD*/XArray* /* of VtsTE */ new_tab, 2706b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID* ii ) 2707b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 2708b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsTE *old_te, *new_te; 2709b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID old_id, new_id; 2710b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* We're relying here on VG_(indexXA)'s range checking to assert on 2711b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov any stupid values, in particular *ii == VtsID_INVALID. */ 2712b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov old_id = *ii; 2713b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov old_te = VG_(indexXA)( old_tab, old_id ); 2714b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov old_te->rc--; 2715b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov new_id = old_te->remap; 2716b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov new_te = VG_(indexXA)( new_tab, new_id ); 2717b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov new_te->rc++; 2718b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov *ii = new_id; 2719b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 2720b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2721b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic 2722b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovvoid remap_VtsIDs_in_SVal ( /*MOD*/XArray* /* of VtsTE */ old_tab, 2723b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /*MOD*/XArray* /* of VtsTE */ new_tab, 2724b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov SVal* s ) 2725b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 2726b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov SVal old_sv, new_sv; 2727b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov old_sv = *s; 2728b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (SVal__isC(old_sv)) { 2729b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID rMin, wMin; 2730b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov rMin = SVal__unC_Rmin(old_sv); 2731b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov wMin = SVal__unC_Wmin(old_sv); 2732b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov remap_VtsID( old_tab, new_tab, &rMin ); 2733b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov remap_VtsID( old_tab, new_tab, &wMin ); 2734b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov new_sv = SVal__mkC( rMin, wMin ); 2735b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov *s = new_sv; 2736b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 2737b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 2738b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2739b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2740ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* NOT TO BE CALLED FROM WITHIN libzsm. */ 2741ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown__attribute__((noinline)) 2742ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void vts_tab__do_GC ( Bool show_stats ) 2743ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2744ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i, nTab, nLive, nFreed; 2745ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2746b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* ---------- BEGIN VTS GC ---------- */ 2747ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* check this is actually necessary. */ 2748ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(vts_tab_freelist == VtsID_INVALID); 2749ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2750ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* empty the caches for partial order checks and binary joins. We 2751ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown could do better and prune out the entries to be deleted, but it 2752ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ain't worth the hassle. */ 2753ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__invalidate_caches(); 2754ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2755ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* First, make the reference counts up to date. */ 2756ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_flush_cache(); 2757ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2758ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nTab = VG_(sizeXA)( vts_tab ); 2759ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2760ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (show_stats) { 2761ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("<<GC begins at vts_tab size %lu>>\n", nTab); 2762ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown show_vts_stats("before GC"); 2763ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2764ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2765b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Now we can inspect the entire vts_tab. Any entries with zero 2766b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov .rc fields are now no longer in use and can be put back on the 2767ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown free list, removed from vts_set, and deleted. */ 2768ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nFreed = 0; 2769ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < nTab; i++) { 2770ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool present; 2771b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UWord oldK = 0, oldV = 12345; 2772ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsTE* te = VG_(indexXA)( vts_tab, i ); 2773ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (te->vts == NULL) { 2774ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(te->rc == 0); 2775ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown continue; /* already on the free list (presumably) */ 2776ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2777ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (te->rc > 0) 2778ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown continue; /* in use */ 2779ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Ok, we got one we can free. */ 2780ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(te->vts->id == i); 2781ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* first, remove it from vts_set. */ 2782ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown present = VG_(delFromFM)( vts_set, 2783ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown &oldK, &oldV, (UWord)te->vts ); 2784ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(present); /* else it isn't in vts_set ?! */ 2785ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(oldV == 0); /* no info stored in vts_set val fields */ 2786ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(oldK == (UWord)te->vts); /* else what did delFromFM find?! */ 2787ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* now free the VTS itself */ 2788ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VTS__delete(te->vts); 2789ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown te->vts = NULL; 2790ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* and finally put this entry on the free list */ 2791ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(te->freelink == VtsID_INVALID); /* can't already be on it */ 2792ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown add_to_free_list( i ); 2793ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nFreed++; 2794ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2795ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2796ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Now figure out when the next GC should be. We'll allow the 2797ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown number of VTSs to double before GCing again. Except of course 2798ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown that since we can't (or, at least, don't) shrink vts_tab, we 2799ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown can't set the threshhold value smaller than it. */ 2800ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(nFreed <= nTab); 2801ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nLive = nTab - nFreed; 2802ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(nLive >= 0 && nLive <= nTab); 2803ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts_next_GC_at = 2 * nLive; 2804ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (vts_next_GC_at < nTab) 2805ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts_next_GC_at = nTab; 2806ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2807ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (show_stats) { 2808ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown show_vts_stats("after GC"); 2809ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("<<GC ends, next gc at %ld>>\n", vts_next_GC_at); 2810ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2811ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2812ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (VG_(clo_stats)) { 2813b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov static UInt ctr = 1; 2814ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(nTab > 0); 2815ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(message)(Vg_DebugMsg, 2816ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "libhb: VTS GC: #%u old size %lu live %lu (%2llu%%)\n", 2817ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ctr++, nTab, nLive, (100ULL * (ULong)nLive) / (ULong)nTab); 2818ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2819b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* ---------- END VTS GC ---------- */ 2820b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2821b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Decide whether to do VTS pruning. We have one of three 2822b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov settings. */ 2823b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov static UInt pruning_auto_ctr = 0; /* do not make non-static */ 2824b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2825b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Bool do_pruning = False; 2826b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov switch (HG_(clo_vts_pruning)) { 2827b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov case 0: /* never */ 2828b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov break; 2829b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov case 1: /* auto */ 2830b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov do_pruning = (++pruning_auto_ctr % 5) == 0; 2831b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov break; 2832b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov case 2: /* always */ 2833b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov do_pruning = True; 2834b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov break; 2835b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov default: 2836b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(0); 2837b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 2838b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2839b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* The rest of this routine only handles pruning, so we can 2840b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov quit at this point if it is not to be done. */ 2841b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (!do_pruning) 2842b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return; 2843b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2844b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* ---------- BEGIN VTS PRUNING ---------- */ 2845b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* We begin by sorting the backing table on its .thr values, so as 2846b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov to (1) check they are unique [else something has gone wrong, 2847b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov since it means we must have seen some Thr* exiting more than 2848b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov once, which can't happen], and (2) so that we can quickly look 2849b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov up the dead-thread entries as we work through the VTSs. */ 2850b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(sortXA)( verydead_thread_table ); 2851b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Sanity check: check for unique .sts.thr values. */ 2852b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UWord nBT = VG_(sizeXA)( verydead_thread_table ); 2853b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (nBT > 0) { 2854b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID thrid1, thrid2; 2855b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid2 = *(ThrID*)VG_(indexXA)( verydead_thread_table, 0 ); 2856b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for (i = 1; i < nBT; i++) { 2857b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid1 = thrid2; 2858b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid2 = *(ThrID*)VG_(indexXA)( verydead_thread_table, i ); 2859b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(thrid1 < thrid2); 2860b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 2861b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 2862b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Ok, so the dead thread table has unique and in-order keys. */ 2863b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2864b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* We will run through the old table, and create a new table and 2865b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov set, at the same time setting the .remap entries in the old 2866b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov table to point to the new entries. Then, visit every VtsID in 2867b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov the system, and replace all of them with new ones, using the 2868b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov .remap entries in the old table. Finally, we can delete the old 2869b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov table and set. */ 2870b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2871b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov XArray* /* of VtsTE */ new_tab 2872b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov = VG_(newXA)( HG_(zalloc), "libhb.vts_tab__do_GC.new_tab", 2873b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov HG_(free), sizeof(VtsTE) ); 2874b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2875b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* WordFM VTS* void */ 2876b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov WordFM* new_set 2877b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov = VG_(newFM)( HG_(zalloc), "libhb.vts_tab__do_GC.new_set", 2878b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov HG_(free), 2879b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov (Word(*)(UWord,UWord))VTS__cmp_structural ); 2880b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2881b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Visit each old VTS. For each one: 2882b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2883b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov * make a pruned version 2884b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2885b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov * search new_set for the pruned version, yielding either 2886b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Nothing (not present) or the new VtsID for it. 2887b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2888b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov * if not present, allocate a new VtsID for it, insert (pruned 2889b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS, new VtsID) in the tree, and set 2890b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov remap_table[old VtsID] = new VtsID. 2891b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2892b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov * if present, set remap_table[old VtsID] = new VtsID, where 2893b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov new VtsID was determined by the tree lookup. Then free up 2894b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov the clone. 2895b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov */ 2896b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2897b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UWord nBeforePruning = 0, nAfterPruning = 0; 2898b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UWord nSTSsBefore = 0, nSTSsAfter = 0; 2899b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID new_VtsID_ctr = 0; 2900b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2901b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for (i = 0; i < nTab; i++) { 2902b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2903b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* For each old VTS .. */ 2904b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsTE* old_te = VG_(indexXA)( vts_tab, i ); 2905b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS* old_vts = old_te->vts; 2906b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(old_te->remap == VtsID_INVALID); 2907b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2908b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Skip it if not in use */ 2909b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (old_te->rc == 0) { 2910b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(old_vts == NULL); 2911b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov continue; 2912b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 2913b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(old_vts != NULL); 2914b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(old_vts->id == i); 2915b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(old_vts->ts != NULL); 2916b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2917b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* It is in use. Make a pruned version. */ 2918b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov nBeforePruning++; 2919b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov nSTSsBefore += old_vts->usedTS; 2920b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS* new_vts = VTS__subtract("libhb.vts_tab__do_GC.new_vts", 2921b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov old_vts, verydead_thread_table); 2922b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(new_vts->sizeTS == new_vts->usedTS); 2923b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(*(ULong*)(&new_vts->ts[new_vts->usedTS]) 2924b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov == 0x0ddC0ffeeBadF00dULL); 2925b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2926b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Get rid of the old VTS and the tree entry. It's a bit more 2927b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov complex to incrementally delete the VTSs now than to nuke 2928b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov them all after we're done, but the upside is that we don't 2929b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov wind up temporarily storing potentially two complete copies 2930b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov of each VTS and hence spiking memory use. */ 2931b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UWord oldK = 0, oldV = 12345; 2932b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Bool present = VG_(delFromFM)( vts_set, 2933b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov &oldK, &oldV, (UWord)old_vts ); 2934b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(present); /* else it isn't in vts_set ?! */ 2935b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(oldV == 0); /* no info stored in vts_set val fields */ 2936b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(oldK == (UWord)old_vts); /* else what did delFromFM find?! */ 2937b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* now free the VTS itself */ 2938b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS__delete(old_vts); 2939b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov old_te->vts = NULL; 2940b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov old_vts = NULL; 2941b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2942b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* NO MENTIONS of old_vts allowed beyond this point. */ 2943b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2944b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Ok, we have the pruned copy in new_vts. See if a 2945b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov structurally identical version is already present in new_set. 2946b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov If so, delete the one we just made and move on; if not, add 2947b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov it. */ 2948b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS* identical_version = NULL; 2949b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UWord valW = 12345; 2950b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (VG_(lookupFM)(new_set, (UWord*)&identical_version, &valW, 2951b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov (UWord)new_vts)) { 2952b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // already have it 2953b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(valW == 0); 2954b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(identical_version != NULL); 2955b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(identical_version != new_vts); 2956b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS__delete(new_vts); 2957b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov new_vts = identical_version; 2958b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(new_vts->id != VtsID_INVALID); 2959b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } else { 2960b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(valW == 12345); 2961b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(identical_version == NULL); 2962b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov new_vts->id = new_VtsID_ctr++; 2963b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Bool b = VG_(addToFM)(new_set, (UWord)new_vts, 0); 2964b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(!b); 2965b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsTE new_te; 2966b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov new_te.vts = new_vts; 2967b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov new_te.rc = 0; 2968b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov new_te.freelink = VtsID_INVALID; 2969b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov new_te.remap = VtsID_INVALID; 2970b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Word j = VG_(addToXA)( new_tab, &new_te ); 2971b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(j <= i); 2972b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(j == new_VtsID_ctr - 1); 2973b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // stats 2974b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov nAfterPruning++; 2975b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov nSTSsAfter += new_vts->usedTS; 2976b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 2977b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov old_te->remap = new_vts->id; 2978b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2979b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } /* for (i = 0; i < nTab; i++) */ 2980b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2981b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* At this point, we have: 2982b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov * the old VTS table, with its .remap entries set, 2983b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov and with all .vts == NULL. 2984b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov * the old VTS tree should be empty, since it and the old VTSs 2985b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov it contained have been incrementally deleted was we worked 2986b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov through the old table. 2987b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov * the new VTS table, with all .rc == 0, all .freelink and .remap 2988b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov == VtsID_INVALID. 2989b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov * the new VTS tree. 2990b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov */ 2991b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert( VG_(sizeFM)(vts_set) == 0 ); 2992b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2993b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Now actually apply the mapping. */ 2994b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Visit all the VtsIDs in the entire system. Where do we expect 2995b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov to find them? 2996b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov (a) in shadow memory -- the LineZs and LineFs 2997b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov (b) in our collection of struct _Thrs. 2998b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov (c) in our collection of struct _SOs. 2999b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Nowhere else, AFAICS. Not in the zsm cache, because that just 3000b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov got invalidated. 3001b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3002b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Using the .remap fields in vts_tab, map each old VtsID to a new 3003b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsID. For each old VtsID, dec its rc; and for each new one, 3004b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov inc it. This sets up the new refcounts, and it also gives a 3005b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cheap sanity check of the old ones: all old refcounts should be 3006b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov zero after this operation. 3007b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov */ 3008b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3009b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Do the mappings for (a) above: iterate over the Primary shadow 3010b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov mem map (WordFM Addr SecMap*). */ 3011b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UWord secmapW = 0; 3012b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(initIterFM)( map_shmem ); 3013b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov while (VG_(nextIterFM)( map_shmem, NULL, &secmapW )) { 3014b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UWord j; 3015b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov SecMap* sm = (SecMap*)secmapW; 3016b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(sm->magic == SecMap_MAGIC); 3017b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Deal with the LineZs */ 3018b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for (i = 0; i < N_SECMAP_ZLINES; i++) { 3019b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov LineZ* lineZ = &sm->linesZ[i]; 3020b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (lineZ->dict[0] == SVal_INVALID) 3021b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov continue; /* not in use -- data is in F rep instead */ 3022b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for (j = 0; j < 4; j++) 3023b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov remap_VtsIDs_in_SVal(vts_tab, new_tab, &lineZ->dict[j]); 3024b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 3025b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Deal with the LineFs */ 3026b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for (i = 0; i < sm->linesF_size; i++) { 3027b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov LineF* lineF = &sm->linesF[i]; 3028b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (!lineF->inUse) 3029b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov continue; 3030b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for (j = 0; j < N_LINE_ARANGE; j++) 3031b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov remap_VtsIDs_in_SVal(vts_tab, new_tab, &lineF->w64s[j]); 3032b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 3033b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 3034b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(doneIterFM)( map_shmem ); 3035b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3036b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Do the mappings for (b) above: visit our collection of struct 3037b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov _Thrs. */ 3038b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Thread* hgthread = get_admin_threads(); 3039b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(hgthread); 3040b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov while (hgthread) { 3041b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Thr* hbthr = hgthread->hbthr; 3042b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(hbthr); 3043b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Threads that are listed in the prunable set have their viR 3044b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov and viW set to VtsID_INVALID, so we can't mess with them. */ 3045b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (hbthr->llexit_done && hbthr->joinedwith_done) { 3046b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(hbthr->viR == VtsID_INVALID); 3047b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(hbthr->viW == VtsID_INVALID); 3048b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov hgthread = hgthread->admin; 3049b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov continue; 3050b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 3051b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov remap_VtsID( vts_tab, new_tab, &hbthr->viR ); 3052b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov remap_VtsID( vts_tab, new_tab, &hbthr->viW ); 3053b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov hgthread = hgthread->admin; 3054b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 3055b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3056b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Do the mappings for (c) above: visit the struct _SOs. */ 3057b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov SO* so = admin_SO; 3058b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov while (so) { 3059b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (so->viR != VtsID_INVALID) 3060b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov remap_VtsID( vts_tab, new_tab, &so->viR ); 3061b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (so->viW != VtsID_INVALID) 3062b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov remap_VtsID( vts_tab, new_tab, &so->viW ); 3063b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov so = so->admin_next; 3064b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 3065b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3066b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* So, we're nearly done (with this incredibly complex operation). 3067b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Check the refcounts for the old VtsIDs all fell to zero, as 3068b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov expected. Any failure is serious. */ 3069b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for (i = 0; i < nTab; i++) { 3070b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsTE* te = VG_(indexXA)( vts_tab, i ); 3071b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(te->vts == NULL); 3072b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* This is the assert proper. Note we're also asserting 3073b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov zeroness for old entries which are unmapped (hence have 3074b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov .remap == VtsID_INVALID). That's OK. */ 3075b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(te->rc == 0); 3076b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 3077b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3078b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Install the new table and set. */ 3079b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(deleteFM)(vts_set, NULL/*kFin*/, NULL/*vFin*/); 3080b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov vts_set = new_set; 3081b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(deleteXA)( vts_tab ); 3082b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov vts_tab = new_tab; 3083b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3084b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* The freelist of vts_tab entries is empty now, because we've 3085b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov compacted all of the live entries at the low end of the 3086b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov table. */ 3087b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov vts_tab_freelist = VtsID_INVALID; 3088b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3089b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Sanity check vts_set and vts_tab. */ 3090b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3091b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Because all the live entries got slid down to the bottom of vts_tab: */ 3092b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert( VG_(sizeXA)( vts_tab ) == VG_(sizeFM)( vts_set )); 3093b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3094b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Assert that the vts_tab and vts_set entries point at each other 3095b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov in the required way */ 3096b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UWord wordK = 0, wordV = 0; 3097b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(initIterFM)( vts_set ); 3098b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov while (VG_(nextIterFM)( vts_set, &wordK, &wordV )) { 3099b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(wordK != 0); 3100b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(wordV == 0); 3101b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS* vts = (VTS*)wordK; 3102b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(vts->id != VtsID_INVALID); 3103b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsTE* te = VG_(indexXA)( vts_tab, vts->id ); 3104b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(te->vts == vts); 3105b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 3106b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(doneIterFM)( vts_set ); 3107b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3108b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Also iterate over the table, and check each entry is 3109b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov plausible. */ 3110b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov nTab = VG_(sizeXA)( vts_tab ); 3111b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for (i = 0; i < nTab; i++) { 3112b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VtsTE* te = VG_(indexXA)( vts_tab, i ); 3113b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(te->vts); 3114b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(te->vts->id == i); 3115b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(te->rc > 0); /* 'cos we just GC'd */ 3116b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(te->freelink == VtsID_INVALID); /* in use */ 3117b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(te->remap == VtsID_INVALID); /* not relevant */ 3118b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 3119b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3120b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* And we're done. Bwahahaha. Ha. Ha. Ha. */ 3121b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (VG_(clo_stats)) { 3122b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov static UInt ctr = 1; 3123b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(nTab > 0); 3124b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(message)( 3125b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Vg_DebugMsg, 3126b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "libhb: VTS PR: #%u before %lu (avg sz %lu) " 3127b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "after %lu (avg sz %lu)\n", 3128b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ctr++, 3129b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov nBeforePruning, nSTSsBefore / (nBeforePruning ? nBeforePruning : 1), 3130b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov nAfterPruning, nSTSsAfter / (nAfterPruning ? nAfterPruning : 1) 3131b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ); 3132b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 3133b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (0) 3134b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(printf)("VTQ: before pruning %lu (avg sz %lu), " 3135b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "after pruning %lu (avg sz %lu)\n", 3136b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov nBeforePruning, nSTSsBefore / nBeforePruning, 3137b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov nAfterPruning, nSTSsAfter / nAfterPruning); 3138b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* ---------- END VTS PRUNING ---------- */ 3139ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3140ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3141ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3142ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 3143ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 3144ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Vts IDs // 3145ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 3146ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 3147ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3148ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown////////////////////////// 3149b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* A temporary, max-sized VTS which is used as a temporary (the first 3150b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov argument) in VTS__singleton, VTS__tick and VTS__join operations. */ 3151b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic VTS* temp_max_sized_VTS = NULL; 3152b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3153b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov////////////////////////// 3154ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic ULong stats__cmpLEQ_queries = 0; 3155ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic ULong stats__cmpLEQ_misses = 0; 3156ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic ULong stats__join2_queries = 0; 3157ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic ULong stats__join2_misses = 0; 3158ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3159ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline UInt ROL32 ( UInt w, Int n ) { 3160ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown w = (w << n) | (w >> (32-n)); 3161ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return w; 3162ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3163ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline UInt hash_VtsIDs ( VtsID vi1, VtsID vi2, UInt nTab ) { 3164ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt hash = ROL32(vi1,19) ^ ROL32(vi2,13); 3165ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return hash % nTab; 3166ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3167ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3168ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_CMPLEQ_CACHE 1023 3169ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 3170ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct { VtsID vi1; VtsID vi2; Bool leq; } 3171ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cmpLEQ_cache[N_CMPLEQ_CACHE]; 3172ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3173ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_JOIN2_CACHE 1023 3174ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 3175ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct { VtsID vi1; VtsID vi2; VtsID res; } 3176ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown join2_cache[N_JOIN2_CACHE]; 3177ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3178ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void VtsID__invalidate_caches ( void ) { 3179ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Int i; 3180ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_CMPLEQ_CACHE; i++) { 3181ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cmpLEQ_cache[i].vi1 = VtsID_INVALID; 3182ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cmpLEQ_cache[i].vi2 = VtsID_INVALID; 3183ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cmpLEQ_cache[i].leq = False; 3184ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3185ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_JOIN2_CACHE; i++) { 3186ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown join2_cache[i].vi1 = VtsID_INVALID; 3187ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown join2_cache[i].vi2 = VtsID_INVALID; 3188ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown join2_cache[i].res = VtsID_INVALID; 3189ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3190ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3191ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown////////////////////////// 3192ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3193ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//static Bool VtsID__is_valid ( VtsID vi ) { 3194ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// VtsTE* ve; 3195ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// if (vi >= (VtsID)VG_(sizeXA)( vts_tab )) 3196ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// return False; 3197ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// ve = VG_(indexXA)( vts_tab, vi ); 3198ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// if (!ve->vts) 3199ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// return False; 3200ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// tl_assert(ve->vts->id == vi); 3201ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// return True; 3202ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//} 3203ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3204ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic VTS* VtsID__to_VTS ( VtsID vi ) { 3205ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsTE* te = VG_(indexXA)( vts_tab, vi ); 3206ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(te->vts); 3207ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return te->vts; 3208ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3209ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3210ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void VtsID__pp ( VtsID vi ) { 3211ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HChar buf[100]; 3212ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VTS* vts = VtsID__to_VTS(vi); 3213ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VTS__show( buf, sizeof(buf)-1, vts ); 3214ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown buf[sizeof(buf)-1] = 0; 3215ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s", buf); 3216ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3217ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3218ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* compute partial ordering relation of vi1 and vi2. */ 3219ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown__attribute__((noinline)) 3220ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Bool VtsID__cmpLEQ_WRK ( VtsID vi1, VtsID vi2 ) { 3221ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt hash; 3222ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool leq; 3223ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VTS *v1, *v2; 3224ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //if (vi1 == vi2) return True; 3225ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(vi1 != vi2); 3226ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ////++ 3227ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cmpLEQ_queries++; 3228ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown hash = hash_VtsIDs(vi1, vi2, N_CMPLEQ_CACHE); 3229ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (cmpLEQ_cache[hash].vi1 == vi1 3230ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown && cmpLEQ_cache[hash].vi2 == vi2) 3231ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return cmpLEQ_cache[hash].leq; 3232ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cmpLEQ_misses++; 3233ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ////-- 3234ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown v1 = VtsID__to_VTS(vi1); 3235ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown v2 = VtsID__to_VTS(vi2); 3236b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov leq = VTS__cmpLEQ( v1, v2 ) == 0; 3237ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ////++ 3238ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cmpLEQ_cache[hash].vi1 = vi1; 3239ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cmpLEQ_cache[hash].vi2 = vi2; 3240ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cmpLEQ_cache[hash].leq = leq; 3241ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ////-- 3242ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return leq; 3243ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3244ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool VtsID__cmpLEQ ( VtsID vi1, VtsID vi2 ) { 3245ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return LIKELY(vi1 == vi2) ? True : VtsID__cmpLEQ_WRK(vi1, vi2); 3246ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3247ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3248ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* compute binary join */ 3249ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown__attribute__((noinline)) 3250ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic VtsID VtsID__join2_WRK ( VtsID vi1, VtsID vi2 ) { 3251ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt hash; 3252ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID res; 3253b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS *vts1, *vts2; 3254ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //if (vi1 == vi2) return vi1; 3255ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(vi1 != vi2); 3256ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ////++ 3257ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__join2_queries++; 3258ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown hash = hash_VtsIDs(vi1, vi2, N_JOIN2_CACHE); 3259ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (join2_cache[hash].vi1 == vi1 3260ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown && join2_cache[hash].vi2 == vi2) 3261ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return join2_cache[hash].res; 3262ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__join2_misses++; 3263ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ////-- 3264ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts1 = VtsID__to_VTS(vi1); 3265ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts2 = VtsID__to_VTS(vi2); 3266b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov temp_max_sized_VTS->usedTS = 0; 3267b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS__join(temp_max_sized_VTS, vts1,vts2); 3268b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov res = vts_tab__find__or__clone_and_add(temp_max_sized_VTS); 3269ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ////++ 3270ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown join2_cache[hash].vi1 = vi1; 3271ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown join2_cache[hash].vi2 = vi2; 3272ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown join2_cache[hash].res = res; 3273ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ////-- 3274ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return res; 3275ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3276ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline VtsID VtsID__join2 ( VtsID vi1, VtsID vi2 ) { 3277ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return LIKELY(vi1 == vi2) ? vi1 : VtsID__join2_WRK(vi1, vi2); 3278ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3279ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3280ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* create a singleton VTS, namely [thr:1] */ 3281ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic VtsID VtsID__mk_Singleton ( Thr* thr, ULong tym ) { 3282b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov temp_max_sized_VTS->usedTS = 0; 3283b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS__singleton(temp_max_sized_VTS, thr,tym); 3284b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return vts_tab__find__or__clone_and_add(temp_max_sized_VTS); 3285ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3286ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3287ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* tick operation, creates value 1 if specified index is absent */ 3288ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic VtsID VtsID__tick ( VtsID vi, Thr* idx ) { 3289ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VTS* vts = VtsID__to_VTS(vi); 3290b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov temp_max_sized_VTS->usedTS = 0; 3291b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS__tick(temp_max_sized_VTS, idx,vts); 3292b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return vts_tab__find__or__clone_and_add(temp_max_sized_VTS); 3293ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3294ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3295ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* index into a VTS (only for assertions) */ 3296ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic ULong VtsID__indexAt ( VtsID vi, Thr* idx ) { 3297ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VTS* vts = VtsID__to_VTS(vi); 3298ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return VTS__indexAt_SLOW( vts, idx ); 3299ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3300ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3301ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Assuming that !cmpLEQ(vi1, vi2), find the index of the first (or 3302ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown any, really) element in vi1 which is pointwise greater-than the 3303ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown corresponding element in vi2. If no such element exists, return 3304ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown NULL. This needs to be fairly quick since it is called every time 3305ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a race is detected. */ 3306ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Thr* VtsID__findFirst_notLEQ ( VtsID vi1, VtsID vi2 ) 3307ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3308ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VTS *vts1, *vts2; 3309b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Thr* diffthr; 3310b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID diffthrid; 3311ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(vi1 != vi2); 3312ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts1 = VtsID__to_VTS(vi1); 3313ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts2 = VtsID__to_VTS(vi2); 3314ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(vts1 != vts2); 3315b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov diffthrid = VTS__cmpLEQ(vts1, vts2); 3316b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov diffthr = Thr__from_ThrID(diffthrid); 3317ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(diffthr); /* else they are LEQ ! */ 3318ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return diffthr; 3319ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3320ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3321ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3322ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 3323ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 3324ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Filters // 3325ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 3326ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 3327ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3328ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Forget everything we know -- clear the filter and let everything 3329ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown through. This needs to be as fast as possible, since it is called 3330ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown every time the running thread changes, and every time a thread's 3331ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vector clocks change, which can be quite frequent. The obvious 3332ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fast way to do this is simply to stuff in tags which we know are 3333ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown not going to match anything, since they're not aligned to the start 3334ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown of a line. */ 3335ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void Filter__clear ( Filter* fi, HChar* who ) 3336ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3337ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i; 3338ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)(" Filter__clear(%p, %s)\n", fi, who); 3339ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < FI_NUM_LINES; i += 8) { 3340ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[i+0] = 1; /* impossible value -- cannot match */ 3341ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[i+1] = 1; 3342ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[i+2] = 1; 3343ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[i+3] = 1; 3344ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[i+4] = 1; 3345ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[i+5] = 1; 3346ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[i+6] = 1; 3347ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[i+7] = 1; 3348ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3349ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(i == FI_NUM_LINES); 3350ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3351ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3352ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Clearing an arbitrary range in the filter. Unfortunately 3353ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown we have to do this due to core-supplied new/die-mem events. */ 3354ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3355ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void Filter__clear_1byte ( Filter* fi, Addr a ) 3356ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3357ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr atag = FI_GET_TAG(a); /* tag of 'a' */ 3358ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord lineno = FI_GET_LINENO(a); /* lineno for 'a' */ 3359ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown FiLine* line = &fi->lines[lineno]; 3360ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord loff = (a - atag) / 8; 3361ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort mask = 0x3 << (2 * (a & 7)); 3362ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* mask is C000, 3000, 0C00, 0300, 00C0, 0030, 000C or 0003 */ 3363ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY( fi->tags[lineno] == atag )) { 3364ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* hit. clear the bits. */ 3365ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort u16 = line->u16s[loff]; 3366ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = u16 & ~mask; /* clear them */ 3367ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 3368ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* miss. The filter doesn't hold this address, so ignore. */ 3369ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3370ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3371ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3372ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void Filter__clear_8bytes_aligned ( Filter* fi, Addr a ) 3373ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3374ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr atag = FI_GET_TAG(a); /* tag of 'a' */ 3375ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord lineno = FI_GET_LINENO(a); /* lineno for 'a' */ 3376ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown FiLine* line = &fi->lines[lineno]; 3377ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord loff = (a - atag) / 8; 3378ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY( fi->tags[lineno] == atag )) { 3379ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = 0; 3380ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 3381ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* miss. The filter doesn't hold this address, so ignore. */ 3382ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3383ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3384ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3385ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void Filter__clear_range ( Filter* fi, Addr a, UWord len ) 3386ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3387ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //VG_(printf)("%lu ", len); 3388ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* slowly do part preceding 8-alignment */ 3389ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (UNLIKELY(!VG_IS_8_ALIGNED(a)) && LIKELY(len > 0)) { 3390ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Filter__clear_1byte( fi, a ); 3391ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a++; 3392ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len--; 3393ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3394ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* vector loop */ 3395ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (len >= 8) { 3396ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Filter__clear_8bytes_aligned( fi, a ); 3397ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 8; 3398ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 8; 3399ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3400ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* slowly do tail */ 3401ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (UNLIKELY(len > 0)) { 3402ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Filter__clear_1byte( fi, a ); 3403ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a++; 3404ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len--; 3405ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3406ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3407ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3408ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3409ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* ------ Read handlers for the filter. ------ */ 3410ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3411ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool Filter__ok_to_skip_crd64 ( Filter* fi, Addr a ) 3412ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3413ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !VG_IS_8_ALIGNED(a) )) 3414ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3415ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown { 3416ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr atag = FI_GET_TAG(a); /* tag of 'a' */ 3417ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord lineno = FI_GET_LINENO(a); /* lineno for 'a' */ 3418ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown FiLine* line = &fi->lines[lineno]; 3419ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord loff = (a - atag) / 8; 3420ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort mask = 0xAAAA; 3421ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY( fi->tags[lineno] == atag )) { 3422ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* hit. check line and update. */ 3423ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort u16 = line->u16s[loff]; 3424ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool ok = (u16 & mask) == mask; /* all R bits set? */ 3425ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = u16 | mask; /* set them */ 3426ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ok; 3427ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 3428ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* miss. nuke existing line and re-use it. */ 3429ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i; 3430ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[lineno] = atag; 3431ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < FI_LINE_SZB / 8; i++) 3432ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[i] = 0; 3433ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = mask; 3434ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3435ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3436ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3437ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3438ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3439ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool Filter__ok_to_skip_crd32 ( Filter* fi, Addr a ) 3440ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3441ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !VG_IS_4_ALIGNED(a) )) 3442ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3443ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown { 3444ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr atag = FI_GET_TAG(a); /* tag of 'a' */ 3445ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord lineno = FI_GET_LINENO(a); /* lineno for 'a' */ 3446ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown FiLine* line = &fi->lines[lineno]; 3447ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord loff = (a - atag) / 8; 3448ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort mask = 0xAA << (2 * (a & 4)); /* 0xAA00 or 0x00AA */ 3449ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY( fi->tags[lineno] == atag )) { 3450ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* hit. check line and update. */ 3451ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort u16 = line->u16s[loff]; 3452ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool ok = (u16 & mask) == mask; /* 4 x R bits set? */ 3453ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = u16 | mask; /* set them */ 3454ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ok; 3455ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 3456ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* miss. nuke existing line and re-use it. */ 3457ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i; 3458ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[lineno] = atag; 3459ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < FI_LINE_SZB / 8; i++) 3460ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[i] = 0; 3461ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = mask; 3462ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3463ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3464ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3465ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3466ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3467ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool Filter__ok_to_skip_crd16 ( Filter* fi, Addr a ) 3468ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3469ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !VG_IS_2_ALIGNED(a) )) 3470ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3471ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown { 3472ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr atag = FI_GET_TAG(a); /* tag of 'a' */ 3473ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord lineno = FI_GET_LINENO(a); /* lineno for 'a' */ 3474ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown FiLine* line = &fi->lines[lineno]; 3475ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord loff = (a - atag) / 8; 3476ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort mask = 0xA << (2 * (a & 6)); 3477ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* mask is A000, 0A00, 00A0 or 000A */ 3478ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY( fi->tags[lineno] == atag )) { 3479ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* hit. check line and update. */ 3480ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort u16 = line->u16s[loff]; 3481ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool ok = (u16 & mask) == mask; /* 2 x R bits set? */ 3482ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = u16 | mask; /* set them */ 3483ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ok; 3484ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 3485ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* miss. nuke existing line and re-use it. */ 3486ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i; 3487ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[lineno] = atag; 3488ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < FI_LINE_SZB / 8; i++) 3489ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[i] = 0; 3490ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = mask; 3491ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3492ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3493ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3494ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3495ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3496ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool Filter__ok_to_skip_crd08 ( Filter* fi, Addr a ) 3497ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3498ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown { 3499ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr atag = FI_GET_TAG(a); /* tag of 'a' */ 3500ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord lineno = FI_GET_LINENO(a); /* lineno for 'a' */ 3501ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown FiLine* line = &fi->lines[lineno]; 3502ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord loff = (a - atag) / 8; 3503ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort mask = 0x2 << (2 * (a & 7)); 3504ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* mask is 8000, 2000, 0800, 0200, 0080, 0020, 0008 or 0002 */ 3505ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY( fi->tags[lineno] == atag )) { 3506ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* hit. check line and update. */ 3507ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort u16 = line->u16s[loff]; 3508ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool ok = (u16 & mask) == mask; /* 1 x R bits set? */ 3509ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = u16 | mask; /* set them */ 3510ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ok; 3511ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 3512ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* miss. nuke existing line and re-use it. */ 3513ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i; 3514ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[lineno] = atag; 3515ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < FI_LINE_SZB / 8; i++) 3516ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[i] = 0; 3517ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = mask; 3518ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3519ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3520ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3521ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3522ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3523ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3524ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* ------ Write handlers for the filter. ------ */ 3525ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3526ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool Filter__ok_to_skip_cwr64 ( Filter* fi, Addr a ) 3527ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3528ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !VG_IS_8_ALIGNED(a) )) 3529ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3530ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown { 3531ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr atag = FI_GET_TAG(a); /* tag of 'a' */ 3532ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord lineno = FI_GET_LINENO(a); /* lineno for 'a' */ 3533ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown FiLine* line = &fi->lines[lineno]; 3534ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord loff = (a - atag) / 8; 3535ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort mask = 0xFFFF; 3536ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY( fi->tags[lineno] == atag )) { 3537ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* hit. check line and update. */ 3538ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort u16 = line->u16s[loff]; 3539ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool ok = (u16 & mask) == mask; /* all R & W bits set? */ 3540ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = u16 | mask; /* set them */ 3541ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ok; 3542ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 3543ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* miss. nuke existing line and re-use it. */ 3544ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i; 3545ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[lineno] = atag; 3546ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < FI_LINE_SZB / 8; i++) 3547ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[i] = 0; 3548ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = mask; 3549ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3550ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3551ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3552ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3553ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3554ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool Filter__ok_to_skip_cwr32 ( Filter* fi, Addr a ) 3555ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3556ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !VG_IS_4_ALIGNED(a) )) 3557ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3558ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown { 3559ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr atag = FI_GET_TAG(a); /* tag of 'a' */ 3560ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord lineno = FI_GET_LINENO(a); /* lineno for 'a' */ 3561ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown FiLine* line = &fi->lines[lineno]; 3562ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord loff = (a - atag) / 8; 3563ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort mask = 0xFF << (2 * (a & 4)); /* 0xFF00 or 0x00FF */ 3564ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY( fi->tags[lineno] == atag )) { 3565ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* hit. check line and update. */ 3566ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort u16 = line->u16s[loff]; 3567ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool ok = (u16 & mask) == mask; /* 4 x R & W bits set? */ 3568ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = u16 | mask; /* set them */ 3569ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ok; 3570ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 3571ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* miss. nuke existing line and re-use it. */ 3572ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i; 3573ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[lineno] = atag; 3574ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < FI_LINE_SZB / 8; i++) 3575ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[i] = 0; 3576ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = mask; 3577ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3578ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3579ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3580ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3581ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3582ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool Filter__ok_to_skip_cwr16 ( Filter* fi, Addr a ) 3583ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3584ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !VG_IS_2_ALIGNED(a) )) 3585ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3586ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown { 3587ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr atag = FI_GET_TAG(a); /* tag of 'a' */ 3588ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord lineno = FI_GET_LINENO(a); /* lineno for 'a' */ 3589ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown FiLine* line = &fi->lines[lineno]; 3590ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord loff = (a - atag) / 8; 3591ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort mask = 0xF << (2 * (a & 6)); 3592ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* mask is F000, 0F00, 00F0 or 000F */ 3593ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY( fi->tags[lineno] == atag )) { 3594ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* hit. check line and update. */ 3595ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort u16 = line->u16s[loff]; 3596ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool ok = (u16 & mask) == mask; /* 2 x R & W bits set? */ 3597ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = u16 | mask; /* set them */ 3598ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ok; 3599ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 3600ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* miss. nuke existing line and re-use it. */ 3601ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i; 3602ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[lineno] = atag; 3603ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < FI_LINE_SZB / 8; i++) 3604ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[i] = 0; 3605ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = mask; 3606ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3607ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3608ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3609ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3610ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3611ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool Filter__ok_to_skip_cwr08 ( Filter* fi, Addr a ) 3612ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3613ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown { 3614ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr atag = FI_GET_TAG(a); /* tag of 'a' */ 3615ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord lineno = FI_GET_LINENO(a); /* lineno for 'a' */ 3616ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown FiLine* line = &fi->lines[lineno]; 3617ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord loff = (a - atag) / 8; 3618ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort mask = 0x3 << (2 * (a & 7)); 3619ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* mask is C000, 3000, 0C00, 0300, 00C0, 0030, 000C or 0003 */ 3620ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY( fi->tags[lineno] == atag )) { 3621ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* hit. check line and update. */ 3622ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort u16 = line->u16s[loff]; 3623ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool ok = (u16 & mask) == mask; /* 1 x R bits set? */ 3624ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = u16 | mask; /* set them */ 3625ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ok; 3626ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 3627ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* miss. nuke existing line and re-use it. */ 3628ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i; 3629ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fi->tags[lineno] = atag; 3630ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < FI_LINE_SZB / 8; i++) 3631ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[i] = 0; 3632ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line->u16s[loff] = mask; 3633ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 3634ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3635ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3636ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3637ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3638ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3639ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 3640ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 3641ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Threads // 3642ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 3643ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 3644ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3645b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Maps ThrID values to their Thr*s (which contain ThrID values that 3646b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov should point back to the relevant slot in the array. Lowest 3647b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov numbered slot (0) is for thrid = 1024, (1) is for 1025, etc. */ 3648b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic XArray* /* of Thr* */ thrid_to_thr_map = NULL; 3649ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3650b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* And a counter to dole out ThrID values. For rationale/background, 3651b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov see comments on definition of ScalarTS (far) above. */ 3652b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic ThrID thrid_counter = 1024; /* runs up to ThrID_MAX_VALID */ 3653ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3654b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic ThrID Thr__to_ThrID ( Thr* thr ) { 3655b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return thr->thrid; 3656b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 3657b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic Thr* Thr__from_ThrID ( UInt thrid ) { 3658b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Thr* thr = *(Thr**)VG_(indexXA)( thrid_to_thr_map, thrid - 1024 ); 3659b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(thr->thrid == thrid); 3660b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return thr; 3661b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 3662ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3663b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic Thr* Thr__new ( void ) 3664b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 3665ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Thr* thr = HG_(zalloc)( "libhb.Thr__new.1", sizeof(Thr) ); 3666ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr->viR = VtsID_INVALID; 3667ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr->viW = VtsID_INVALID; 3668b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thr->llexit_done = False; 3669b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thr->joinedwith_done = False; 3670ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr->filter = HG_(zalloc)( "libhb.Thr__new.2", sizeof(Filter) ); 3671ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* We only really need this at history level 1, but unfortunately 3672ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown this routine is called before the command line processing is 3673ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown done (sigh), so we can't rely on HG_(clo_history_level) at this 3674ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown point. Hence always allocate it. Bah. */ 3675ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr->local_Kws_n_stacks 3676ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown = VG_(newXA)( HG_(zalloc), 3677ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "libhb.Thr__new.3 (local_Kws_and_stacks)", 3678ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free), sizeof(ULong_n_EC) ); 3679b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3680b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Add this Thr* <-> ThrID binding to the mapping, and 3681b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cross-check */ 3682b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (!thrid_to_thr_map) { 3683b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thrid_to_thr_map = VG_(newXA)( HG_(zalloc), "libhb.Thr__new.4", 3684b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov HG_(free), sizeof(Thr*) ); 3685b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(thrid_to_thr_map); 3686b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 3687b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3688b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (thrid_counter >= ThrID_MAX_VALID) { 3689b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* We're hosed. We have to stop. */ 3690b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov scalarts_limitations_fail_NORETURN( True/*due_to_nThrs*/ ); 3691b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 3692b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3693b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thr->thrid = thrid_counter++; 3694b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Word ix = VG_(addToXA)( thrid_to_thr_map, &thr ); 3695b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(ix + 1024 == thr->thrid); 3696b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 3697ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return thr; 3698ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3699ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3700ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void note_local_Kw_n_stack_for ( Thr* thr ) 3701ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3702ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word nPresent; 3703ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ULong_n_EC pair; 3704ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(thr); 3705ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3706ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // We only collect this info at history level 1 (approx) 3707ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (HG_(clo_history_level) != 1) 3708ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 3709ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3710ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* This is the scalar Kw for thr. */ 3711ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown pair.ull = VtsID__indexAt( thr->viW, thr ); 3712ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown pair.ec = main_get_EC( thr ); 3713ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(pair.ec); 3714ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(thr->local_Kws_n_stacks); 3715ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3716ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* check that we're not adding duplicates */ 3717ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nPresent = VG_(sizeXA)( thr->local_Kws_n_stacks ); 3718ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3719ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Throw away old stacks, if necessary. We can't accumulate stuff 3720ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown indefinitely. */ 3721ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (nPresent >= N_KWs_N_STACKs_PER_THREAD) { 3722ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(dropHeadXA)( thr->local_Kws_n_stacks, nPresent / 2 ); 3723ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nPresent = VG_(sizeXA)( thr->local_Kws_n_stacks ); 3724ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) 3725ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("LOCAL Kw: thr %p, Kw %llu, ec %p (!!! gc !!!)\n", 3726ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr, pair.ull, pair.ec ); 3727ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3728ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3729ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (nPresent > 0) { 3730ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ULong_n_EC* prevPair 3731ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown = (ULong_n_EC*)VG_(indexXA)( thr->local_Kws_n_stacks, nPresent-1 ); 3732ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert( prevPair->ull <= pair.ull ); 3733ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3734ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3735ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (nPresent == 0) 3736ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown pair.ec = NULL; 3737ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3738ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(addToXA)( thr->local_Kws_n_stacks, &pair ); 3739ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3740ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) 3741ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("LOCAL Kw: thr %p, Kw %llu, ec %p\n", 3742ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr, pair.ull, pair.ec ); 3743ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) 3744ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(pp_ExeContext)(pair.ec); 3745ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3746ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3747ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Int cmp__ULong_n_EC__by_ULong ( ULong_n_EC* pair1, ULong_n_EC* pair2 ) 3748ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3749ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (pair1->ull < pair2->ull) return -1; 3750ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (pair1->ull > pair2->ull) return 1; 3751ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0; 3752ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3753ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3754ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3755ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 3756ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 3757ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Shadow Values // 3758ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 3759ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 3760ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3761ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// type SVal, SVal_INVALID and SVal_NOACCESS are defined by 3762ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// hb_zsm.h. We have to do everything else here. 3763ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3764ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* SVal is 64 bit unsigned int. 3765ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3766ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown <---------30---------> <---------30---------> 3767ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 00 X-----Rmin-VtsID-----X 00 X-----Wmin-VtsID-----X C(Rmin,Wmin) 3768ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 10 X--------------------X XX X--------------------X A: SVal_NOACCESS 3769ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 11 0--------------------0 00 0--------------------0 A: SVal_INVALID 3770ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3771ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 3772ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define SVAL_TAGMASK (3ULL << 62) 3773ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3774ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool SVal__isC ( SVal s ) { 3775ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return (0ULL << 62) == (s & SVAL_TAGMASK); 3776ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3777ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline SVal SVal__mkC ( VtsID rmini, VtsID wmini ) { 3778ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //tl_assert(VtsID__is_valid(rmini)); 3779ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //tl_assert(VtsID__is_valid(wmini)); 3780ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return (((ULong)rmini) << 32) | ((ULong)wmini); 3781ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3782ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline VtsID SVal__unC_Rmin ( SVal s ) { 3783ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(SVal__isC(s)); 3784ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return (VtsID)(s >> 32); 3785ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3786ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline VtsID SVal__unC_Wmin ( SVal s ) { 3787ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(SVal__isC(s)); 3788ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return (VtsID)(s & 0xFFFFFFFFULL); 3789ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3790ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3791ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool SVal__isA ( SVal s ) { 3792ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return (2ULL << 62) == (s & SVAL_TAGMASK); 3793ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3794ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline SVal SVal__mkA ( void ) { 3795ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 2ULL << 62; 3796ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3797ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3798ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Direct callback from lib_zsm. */ 3799ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void SVal__rcinc ( SVal s ) { 3800ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (SVal__isC(s)) { 3801ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc( SVal__unC_Rmin(s) ); 3802ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc( SVal__unC_Wmin(s) ); 3803ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3804ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3805ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3806ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Direct callback from lib_zsm. */ 3807ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void SVal__rcdec ( SVal s ) { 3808ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (SVal__isC(s)) { 3809ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcdec( SVal__unC_Rmin(s) ); 3810ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcdec( SVal__unC_Wmin(s) ); 3811ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3812ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3813ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3814ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3815ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 3816ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 3817ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// A simple group (memory) allocator // 3818ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 3819ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 3820ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3821ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//////////////// BEGIN general group allocator 3822ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef 3823ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct { 3824ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord elemSzB; /* element size */ 3825ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord nPerGroup; /* # elems per group */ 3826ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown void* (*alloc)(HChar*, SizeT); /* group allocator */ 3827ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HChar* cc; /* group allocator's cc */ 3828ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown void (*free)(void*); /* group allocator's free-er (unused) */ 3829ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* XArray of void* (pointers to groups). The groups themselves. 3830ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Each element is a pointer to a block of size (elemSzB * 3831ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nPerGroup) bytes. */ 3832ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown XArray* groups; 3833ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* next free element. Is a pointer to an element in one of the 3834ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown groups pointed to by .groups. */ 3835ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown void* nextFree; 3836ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3837ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown GroupAlloc; 3838ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3839ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void init_GroupAlloc ( /*MOD*/GroupAlloc* ga, 3840ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord elemSzB, 3841ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord nPerGroup, 3842ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown void* (*alloc)(HChar*, SizeT), 3843ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HChar* cc, 3844ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown void (*free)(void*) ) 3845ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3846ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0 == (elemSzB % sizeof(UWord))); 3847ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(elemSzB >= sizeof(UWord)); 3848ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(nPerGroup >= 100); /* let's say */ 3849ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(alloc); 3850ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(cc); 3851ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(free); 3852ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ga); 3853ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(memset)(ga, 0, sizeof(*ga)); 3854ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ga->elemSzB = elemSzB; 3855ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ga->nPerGroup = nPerGroup; 3856ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ga->groups = NULL; 3857ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ga->alloc = alloc; 3858ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ga->cc = cc; 3859ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ga->free = free; 3860ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ga->groups = VG_(newXA)( alloc, cc, free, sizeof(void*) ); 3861ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ga->nextFree = NULL; 3862ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ga->groups); 3863ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3864ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3865ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* The freelist is empty. Allocate a new group and put all the new 3866ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown elements in it onto the freelist. */ 3867ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown__attribute__((noinline)) 3868ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void gal_add_new_group ( GroupAlloc* ga ) 3869ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3870ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i; 3871ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord* group; 3872ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ga); 3873ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ga->nextFree == NULL); 3874ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown group = ga->alloc( ga->cc, ga->elemSzB * ga->nPerGroup ); 3875ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(group); 3876ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* extend the freelist through the new group. Place the freelist 3877ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown pointer in the first word of each element. That's why the 3878ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown element size must be at least one word. */ 3879ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = ga->nPerGroup-1; i >= 0; i--) { 3880ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UChar* elemC = ((UChar*)group) + i * ga->elemSzB; 3881ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord* elem = (UWord*)elemC; 3882ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0 == (((UWord)elem) % sizeof(UWord))); 3883ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *elem = (UWord)ga->nextFree; 3884ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ga->nextFree = elem; 3885ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3886ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* and add to our collection of groups */ 3887ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(addToXA)( ga->groups, &group ); 3888ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3889ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3890ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browninline static void* gal_Alloc ( GroupAlloc* ga ) 3891ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3892ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord* elem; 3893ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(ga->nextFree == NULL)) { 3894ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown gal_add_new_group(ga); 3895ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 3896ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown elem = ga->nextFree; 3897ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ga->nextFree = (void*)*elem; 3898ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *elem = 0; /* unnecessary, but just to be on the safe side */ 3899ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return elem; 3900ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3901ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3902ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browninline static void* gal_Alloc_w_size_check ( GroupAlloc* ga, SizeT n ) 3903ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3904ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(n == ga->elemSzB); 3905ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return gal_Alloc( ga ); 3906ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3907ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3908ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browninline static void gal_Free ( GroupAlloc* ga, void* p ) 3909ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 3910ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord* elem = (UWord*)p; 3911ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *elem = (UWord)ga->nextFree; 3912ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ga->nextFree = elem; 3913ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 3914ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//////////////// END general group allocator 3915ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3916ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3917ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 3918ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 3919ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Change-event map2 // 3920ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 3921ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 3922ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3923ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define EVENT_MAP_GC_DISCARD_FRACTION 0.5 3924ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3925ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* This is in two parts: 3926ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3927ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1. A hash table of RCECs. This is a set of reference-counted stack 3928ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown traces. When the reference count of a stack trace becomes zero, 3929ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown it is removed from the set and freed up. The intent is to have 3930ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a set of stack traces which can be referred to from (2), but to 3931ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown only represent each one once. The set is indexed/searched by 3932ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ordering on the stack trace vectors. 3933ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3934ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2. A SparseWA of OldRefs. These store information about each old 3935ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ref that we need to record. It is indexed by address of the 3936ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown location for which the information is recorded. For LRU 3937ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown purposes, each OldRef also contains a generation number, 3938ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown indicating when it was most recently accessed. 3939ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3940ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown The important part of an OldRef is, however, its accs[] array. 3941ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown This is an array of N_OLDREF_ACCS which binds (thread, R/W, 3942ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown size) triples to RCECs. This allows us to collect the last 3943ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown access-traceback by up to N_OLDREF_ACCS different triples for 3944ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown this location. The accs[] array is a MTF-array. If a binding 3945ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown falls off the end, that's too bad -- we will lose info about 3946ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown that triple's access to this location. 3947ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3948ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown When the SparseWA becomes too big, we can throw away the OldRefs 3949ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown whose generation numbers are below some threshold; hence doing 3950ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown approximate LRU discarding. For each discarded OldRef we must 3951ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown of course decrement the reference count on the all RCECs it 3952ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown refers to, in order that entries from (1) eventually get 3953ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown discarded too. 3954ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3955ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown A major improvement in reliability of this mechanism would be to 3956ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown have a dynamically sized OldRef.accs[] array, so no entries ever 3957ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fall off the end. In investigations (Dec 08) it appears that a 3958ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown major cause for the non-availability of conflicting-access traces 3959ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown in race reports is caused by the fixed size of this array. I 3960ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown suspect for most OldRefs, only a few entries are used, but for a 3961ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown minority of cases there is an overflow, leading to info lossage. 3962ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Investigations also suggest this is very workload and scheduling 3963ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sensitive. Therefore a dynamic sizing would be better. 3964ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3965ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown However, dynamic sizing would defeat the use of a GroupAllocator 3966ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for OldRef structures. And that's important for performance. So 3967ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown it's not straightforward to do. 3968ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 3969ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3970ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3971ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__ctxt_rcdec1 = 0; 3972ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__ctxt_rcdec2 = 0; 3973ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__ctxt_rcdec3 = 0; 3974ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__ctxt_rcdec_calls = 0; 3975ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__ctxt_rcdec_discards = 0; 3976ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__ctxt_rcdec1_eq = 0; 3977ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3978ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__ctxt_tab_curr = 0; 3979ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__ctxt_tab_max = 0; 3980ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3981ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__ctxt_tab_qs = 0; 3982ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__ctxt_tab_cmps = 0; 3983ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3984ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3985ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/////////////////////////////////////////////////////// 3986ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//// Part (1): A hash table of RCECs 3987ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/// 3988ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3989ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_FRAMES 8 3990ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3991ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// (UInt) `echo "Reference Counted Execution Context" | md5sum` 3992ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define RCEC_MAGIC 0xab88abb2UL 3993ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3994ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//#define N_RCEC_TAB 98317 /* prime */ 3995ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_RCEC_TAB 196613 /* prime */ 3996ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 3997ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef 3998ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct _RCEC { 3999ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord magic; /* sanity check only */ 4000ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct _RCEC* next; 4001ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord rc; 4002ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord rcX; /* used for crosschecking */ 4003ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord frames_hash; /* hash of all the frames */ 4004ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord frames[N_FRAMES]; 4005ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4006ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown RCEC; 4007ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4008ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic RCEC** contextTab = NULL; /* hash table of RCEC*s */ 4009ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4010ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4011ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Gives an arbitrary total order on RCEC .frames fields */ 4012ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Word RCEC__cmp_by_frames ( RCEC* ec1, RCEC* ec2 ) { 4013ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i; 4014ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ec1 && ec1->magic == RCEC_MAGIC); 4015ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ec2 && ec2->magic == RCEC_MAGIC); 4016ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (ec1->frames_hash < ec2->frames_hash) return -1; 4017ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (ec1->frames_hash > ec2->frames_hash) return 1; 4018ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_FRAMES; i++) { 4019ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (ec1->frames[i] < ec2->frames[i]) return -1; 4020ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (ec1->frames[i] > ec2->frames[i]) return 1; 4021ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4022ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0; 4023ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4024ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4025ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4026ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Dec the ref of this RCEC. */ 4027ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void ctxt__rcdec ( RCEC* ec ) 4028ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 4029ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_rcdec_calls++; 4030ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ec && ec->magic == RCEC_MAGIC); 4031ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ec->rc > 0); 4032ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ec->rc--; 4033ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4034ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4035ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void ctxt__rcinc ( RCEC* ec ) 4036ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 4037ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ec && ec->magic == RCEC_MAGIC); 4038ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ec->rc++; 4039ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4040ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4041ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4042ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//////////// BEGIN RCEC group allocator 4043ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic GroupAlloc rcec_group_allocator; 4044ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4045ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic RCEC* alloc_RCEC ( void ) { 4046ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return gal_Alloc ( &rcec_group_allocator ); 4047ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4048ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4049ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void free_RCEC ( RCEC* rcec ) { 4050ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(rcec->magic == RCEC_MAGIC); 4051ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown gal_Free( &rcec_group_allocator, rcec ); 4052ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4053ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//////////// END RCEC group allocator 4054ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4055ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4056ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Find 'ec' in the RCEC list whose head pointer lives at 'headp' and 4057ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown move it one step closer the the front of the list, so as to make 4058ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown subsequent searches for it cheaper. */ 4059ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void move_RCEC_one_step_forward ( RCEC** headp, RCEC* ec ) 4060ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 4061ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown RCEC *ec0, *ec1, *ec2; 4062ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (ec == *headp) 4063ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); /* already at head of list */ 4064ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ec != NULL); 4065ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ec0 = *headp; 4066ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ec1 = NULL; 4067ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ec2 = NULL; 4068ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (True) { 4069ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (ec0 == NULL || ec0 == ec) break; 4070ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ec2 = ec1; 4071ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ec1 = ec0; 4072ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ec0 = ec0->next; 4073ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4074ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ec0 == ec); 4075ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (ec0 != NULL && ec1 != NULL && ec2 != NULL) { 4076ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown RCEC* tmp; 4077ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* ec0 points to ec, ec1 to its predecessor, and ec2 to ec1's 4078ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown predecessor. Swap ec0 and ec1, that is, move ec0 one step 4079ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown closer to the start of the list. */ 4080ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ec2->next == ec1); 4081ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ec1->next == ec0); 4082ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tmp = ec0->next; 4083ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ec2->next = ec0; 4084ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ec0->next = ec1; 4085ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ec1->next = tmp; 4086ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4087ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown else 4088ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (ec0 != NULL && ec1 != NULL && ec2 == NULL) { 4089ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* it's second in the list. */ 4090ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(*headp == ec1); 4091ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ec1->next == ec0); 4092ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ec1->next = ec0->next; 4093ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ec0->next = ec1; 4094ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *headp = ec0; 4095ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4096ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4097ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4098ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4099ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Find the given RCEC in the tree, and return a pointer to it. Or, 4100ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if not present, add the given one to the tree (by making a copy of 4101ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown it, so the caller can immediately deallocate the original) and 4102ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return a pointer to the copy. The caller can safely have 'example' 4103ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown on its stack, since we will always return a pointer to a copy of 4104ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown it, not to the original. Note that the inserted node will have .rc 4105ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown of zero and so the caller must immediatly increment it. */ 4106ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown__attribute__((noinline)) 4107ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic RCEC* ctxt__find_or_add ( RCEC* example ) 4108ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 4109ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord hent; 4110ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown RCEC* copy; 4111ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(example && example->magic == RCEC_MAGIC); 4112ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(example->rc == 0); 4113ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4114ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Search the hash table to see if we already have it. */ 4115ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_tab_qs++; 4116ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown hent = example->frames_hash % N_RCEC_TAB; 4117ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown copy = contextTab[hent]; 4118ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (1) { 4119ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!copy) break; 4120ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(copy->magic == RCEC_MAGIC); 4121ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_tab_cmps++; 4122ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0 == RCEC__cmp_by_frames(copy, example)) break; 4123ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown copy = copy->next; 4124ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4125ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4126ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (copy) { 4127ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(copy != example); 4128ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* optimisation: if it's not at the head of its list, move 1 4129ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown step fwds, to make future searches cheaper */ 4130ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (copy != contextTab[hent]) { 4131ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown move_RCEC_one_step_forward( &contextTab[hent], copy ); 4132ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4133ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 4134ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown copy = alloc_RCEC(); 4135ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(copy != example); 4136ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *copy = *example; 4137ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown copy->next = contextTab[hent]; 4138ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown contextTab[hent] = copy; 4139ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_tab_curr++; 4140ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (stats__ctxt_tab_curr > stats__ctxt_tab_max) 4141ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_tab_max = stats__ctxt_tab_curr; 4142ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4143ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return copy; 4144ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4145ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4146ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline UWord ROLW ( UWord w, Int n ) 4147ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 4148ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Int bpw = 8 * sizeof(UWord); 4149ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown w = (w << n) | (w >> (bpw-n)); 4150ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return w; 4151ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4152ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4153ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown__attribute__((noinline)) 4154ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic RCEC* get_RCEC ( Thr* thr ) 4155ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 4156ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord hash, i; 4157ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown RCEC example; 4158ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown example.magic = RCEC_MAGIC; 4159ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown example.rc = 0; 4160ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown example.rcX = 0; 4161ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown main_get_stacktrace( thr, &example.frames[0], N_FRAMES ); 4162ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown hash = 0; 4163ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_FRAMES; i++) { 4164ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown hash ^= example.frames[i]; 4165ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown hash = ROLW(hash, 19); 4166ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4167ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown example.frames_hash = hash; 4168ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ctxt__find_or_add( &example ); 4169ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4170ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4171ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/////////////////////////////////////////////////////// 4172ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//// Part (2): 4173ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/// A SparseWA guest-addr -> OldRef, that refers to (1) 4174ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/// 4175ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4176ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// (UInt) `echo "Old Reference Information" | md5sum` 4177ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define OldRef_MAGIC 0x30b1f075UL 4178ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4179b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Records an access: a thread, a context (size & writeness) and the 4180b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov number of held locks. The size (1,2,4,8) is encoded as 00 = 1, 01 = 4181b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2, 10 = 4, 11 = 8. 4182ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 4183b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovtypedef 4184b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov struct { 4185b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov RCEC* rcec; 4186b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov WordSetID locksHeldW; 4187b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt thrid : SCALARTS_N_THRBITS; 4188b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt szLg2B : 2; 4189b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt isW : 1; 4190b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 4191b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Thr_n_RCEC; 4192ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4193ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_OLDREF_ACCS 5 4194ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4195ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef 4196ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct { 4197ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord magic; /* sanity check only */ 4198ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord gen; /* when most recently accessed */ 4199ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* or free list when not in use */ 4200b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* unused slots in this array have .thrid == 0, which is invalid */ 4201ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Thr_n_RCEC accs[N_OLDREF_ACCS]; 4202ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4203ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown OldRef; 4204ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4205ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4206ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//////////// BEGIN OldRef group allocator 4207ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic GroupAlloc oldref_group_allocator; 4208ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4209ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic OldRef* alloc_OldRef ( void ) { 4210ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return gal_Alloc ( &oldref_group_allocator ); 4211ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4212ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4213ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void free_OldRef ( OldRef* r ) { 4214ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(r->magic == OldRef_MAGIC); 4215ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown gal_Free( &oldref_group_allocator, r ); 4216ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4217ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//////////// END OldRef group allocator 4218ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4219ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4220ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic SparseWA* oldrefTree = NULL; /* SparseWA* OldRef* */ 4221ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord oldrefGen = 0; /* current LRU generation # */ 4222ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord oldrefTreeN = 0; /* # elems in oldrefTree */ 4223ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord oldrefGenIncAt = 0; /* inc gen # when size hits this */ 4224ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4225ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browninline static UInt min_UInt ( UInt a, UInt b ) { 4226ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return a < b ? a : b; 4227ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4228ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4229ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Compare the intervals [a1,a1+n1) and [a2,a2+n2). Return -1 if the 4230ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown first interval is lower, 1 if the first interval is higher, and 0 4231ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if there is any overlap. Redundant paranoia with casting is there 4232ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown following what looked distinctly like a bug in gcc-4.1.2, in which 4233ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown some of the comparisons were done signedly instead of 4234ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown unsignedly. */ 4235ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Copied from exp-ptrcheck/sg_main.c */ 4236ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Word cmp_nonempty_intervals ( Addr a1, SizeT n1, 4237ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr a2, SizeT n2 ) { 4238ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord a1w = (UWord)a1; 4239ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord n1w = (UWord)n1; 4240ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord a2w = (UWord)a2; 4241ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord n2w = (UWord)n2; 4242ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(n1w > 0 && n2w > 0); 4243ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (a1w + n1w <= a2w) return -1L; 4244ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (a2w + n2w <= a1w) return 1L; 4245ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0; 4246ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4247ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4248ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void event_map_bind ( Addr a, SizeT szB, Bool isW, Thr* thr ) 4249ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 4250ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown OldRef* ref; 4251ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown RCEC* rcec; 4252ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i, j; 4253ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord keyW, valW; 4254ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool b; 4255ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4256b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(thr); 4257b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID thrid = thr->thrid; 4258b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(thrid != 0); /* zero is used to denote an empty slot. */ 4259b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 4260b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov WordSetID locksHeldW = thr->hgthread->locksetW; 4261b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 4262ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcec = get_RCEC( thr ); 4263ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ctxt__rcinc(rcec); 4264ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4265b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt szLg2B = 0; 4266ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown switch (szB) { 4267ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* This doesn't look particularly branch-predictor friendly. */ 4268b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov case 1: szLg2B = 0; break; 4269b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov case 2: szLg2B = 1; break; 4270b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov case 4: szLg2B = 2; break; 4271b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov case 8: szLg2B = 3; break; 4272ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown default: tl_assert(0); 4273ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4274ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4275b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Look in the map to see if we already have a record for this 4276b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov address. */ 4277ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b = VG_(lookupSWA)( oldrefTree, &keyW, &valW, a ); 4278ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4279ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (b) { 4280ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4281ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* We already have a record for this address. We now need to 4282b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov see if we have a stack trace pertaining to this (thrid, R/W, 4283ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown size) triple. */ 4284ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(keyW == a); 4285ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ref = (OldRef*)valW; 4286ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ref->magic == OldRef_MAGIC); 4287ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4288ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_OLDREF_ACCS; i++) { 4289b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (ref->accs[i].thrid != thrid) 4290b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov continue; 4291b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (ref->accs[i].szLg2B != szLg2B) 4292ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown continue; 4293b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (ref->accs[i].isW != (UInt)(isW & 1)) 4294ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown continue; 4295ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* else we have a match, so stop looking. */ 4296ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 4297ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4298ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4299ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (i < N_OLDREF_ACCS) { 4300b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* thread 'thr' has an entry at index 'i'. Update its RCEC. */ 4301ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (i > 0) { 4302ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Thr_n_RCEC tmp = ref->accs[i-1]; 4303ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ref->accs[i-1] = ref->accs[i]; 4304ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ref->accs[i] = tmp; 4305ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown i--; 4306ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4307ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (rcec == ref->accs[i].rcec) stats__ctxt_rcdec1_eq++; 4308ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_rcdec1++; 4309b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ctxt__rcdec( ref->accs[i].rcec ); 4310b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(ref->accs[i].thrid == thrid); 4311b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Update the RCEC and the W-held lockset. */ 4312b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[i].rcec = rcec; 4313b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[i].locksHeldW = locksHeldW; 4314ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 4315b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* No entry for this (thread, R/W, size, nWHeld) quad. 4316b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Shuffle all of them down one slot, and put the new entry 4317b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov at the start of the array. */ 4318b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (ref->accs[N_OLDREF_ACCS-1].thrid != 0) { 4319ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* the last slot is in use. We must dec the rc on the 4320ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown associated rcec. */ 4321ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ref->accs[N_OLDREF_ACCS-1].rcec); 4322ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_rcdec2++; 4323ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0 && 0 == (stats__ctxt_rcdec2 & 0xFFF)) 4324ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("QQQQ %lu overflows\n",stats__ctxt_rcdec2); 4325b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ctxt__rcdec( ref->accs[N_OLDREF_ACCS-1].rcec ); 4326ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 4327ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(!ref->accs[N_OLDREF_ACCS-1].rcec); 4328ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4329ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (j = N_OLDREF_ACCS-1; j >= 1; j--) 4330ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ref->accs[j] = ref->accs[j-1]; 4331b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[0].thrid = thrid; 4332b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[0].szLg2B = szLg2B; 4333b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[0].isW = (UInt)(isW & 1); 4334b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[0].locksHeldW = locksHeldW; 4335b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[0].rcec = rcec; 4336b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* thrid==0 is used to signify an empty slot, so we can't 4337b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov add zero thrid (such a ThrID is invalid anyway). */ 4338b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* tl_assert(thrid != 0); */ /* There's a dominating assert above. */ 4339ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4340ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4341ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ref->gen = oldrefGen; 4342ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4343ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 4344ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4345ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* We don't have a record for this address. Create a new one. */ 4346ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (oldrefTreeN >= oldrefGenIncAt) { 4347ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldrefGen++; 4348ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldrefGenIncAt = oldrefTreeN + 50000; 4349ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)("oldrefTree: new gen %lu at size %lu\n", 4350ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldrefGen, oldrefTreeN ); 4351ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4352ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4353ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ref = alloc_OldRef(); 4354ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ref->magic = OldRef_MAGIC; 4355b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->gen = oldrefGen; 4356b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[0].thrid = thrid; 4357b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[0].szLg2B = szLg2B; 4358b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[0].isW = (UInt)(isW & 1); 4359b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[0].locksHeldW = locksHeldW; 4360b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[0].rcec = rcec; 4361b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 4362b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* thrid==0 is used to signify an empty slot, so we can't 4363b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov add zero thrid (such a ThrID is invalid anyway). */ 4364b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* tl_assert(thrid != 0); */ /* There's a dominating assert above. */ 4365b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 4366b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Clear out the rest of the entries */ 4367ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (j = 1; j < N_OLDREF_ACCS; j++) { 4368b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[j].rcec = NULL; 4369b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[j].thrid = 0; 4370b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[j].szLg2B = 0; 4371b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[j].isW = 0; 4372b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ref->accs[j].locksHeldW = 0; 4373ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4374ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(addToSWA)( oldrefTree, a, (UWord)ref ); 4375ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldrefTreeN++; 4376ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4377ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4378ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4379ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4380ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4381b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Extract info from the conflicting-access machinery. */ 4382ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownBool libhb_event_map_lookup ( /*OUT*/ExeContext** resEC, 4383b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /*OUT*/Thr** resThr, 4384b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /*OUT*/SizeT* resSzB, 4385b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /*OUT*/Bool* resIsW, 4386b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /*OUT*/WordSetID* locksHeldW, 4387ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Thr* thr, Addr a, SizeT szB, Bool isW ) 4388ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 4389ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i, j; 4390ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown OldRef* ref; 4391ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord keyW, valW; 4392ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool b; 4393ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4394b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID cand_thrid; 4395b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov RCEC* cand_rcec; 4396b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Bool cand_isW; 4397b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov SizeT cand_szB; 4398b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov WordSetID cand_locksHeldW; 4399b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Addr cand_a; 4400ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4401ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr toCheck[15]; 4402ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Int nToCheck = 0; 4403ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4404ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(thr); 4405ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(szB == 8 || szB == 4 || szB == 2 || szB == 1); 4406ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4407b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID thrid = thr->thrid; 4408b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 4409ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown toCheck[nToCheck++] = a; 4410ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = -7; i < (Word)szB; i++) { 4411ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (i != 0) 4412ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown toCheck[nToCheck++] = a + i; 4413ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4414ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(nToCheck <= 15); 4415ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4416ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Now see if we can find a suitable matching event for 4417ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown any of the addresses in toCheck[0 .. nToCheck-1]. */ 4418ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (j = 0; j < nToCheck; j++) { 4419ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4420ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cand_a = toCheck[j]; 4421ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // VG_(printf)("test %ld %p\n", j, cand_a); 4422ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4423ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b = VG_(lookupSWA)( oldrefTree, &keyW, &valW, cand_a ); 4424ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!b) 4425ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown continue; 4426ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4427ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ref = (OldRef*)valW; 4428ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(keyW == cand_a); 4429ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ref->magic == OldRef_MAGIC); 4430b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(ref->accs[0].thrid != 0); /* first slot must always be used */ 4431ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4432b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cand_thrid = 0; /* invalid; see comments in event_map_bind */ 4433b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cand_rcec = NULL; 4434b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cand_isW = False; 4435b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cand_szB = 0; 4436b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cand_locksHeldW = 0; /* always valid; see initialise_data_structures() */ 4437ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4438ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_OLDREF_ACCS; i++) { 4439ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Thr_n_RCEC* cand = &ref->accs[i]; 4440b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cand_rcec = cand->rcec; 4441b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cand_thrid = cand->thrid; 4442b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cand_isW = (Bool)cand->isW; 4443b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cand_szB = 1 << cand->szLg2B; 4444b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cand_locksHeldW = cand->locksHeldW; 4445ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4446b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (cand_thrid == 0) 4447ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* This slot isn't in use. Ignore it. */ 4448ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown continue; 4449ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4450b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (cand_thrid == thrid) 4451ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* This is an access by the same thread, but we're only 4452ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown interested in accesses from other threads. Ignore. */ 4453ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown continue; 4454ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4455ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if ((!cand_isW) && (!isW)) 4456ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* We don't want to report a read racing against another 4457ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown read; that's stupid. So in this case move on. */ 4458ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown continue; 4459ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4460ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (cmp_nonempty_intervals(a, szB, cand_a, cand_szB) != 0) 4461ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* No overlap with the access we're asking about. Ignore. */ 4462ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown continue; 4463ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4464ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* We have a match. Stop searching. */ 4465ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 4466ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4467ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4468ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(i >= 0 && i <= N_OLDREF_ACCS); 4469ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4470ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (i < N_OLDREF_ACCS) { 4471ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Int n, maxNFrames; 4472ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* return with success */ 4473b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(cand_thrid); 4474ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(cand_rcec); 4475ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(cand_rcec->magic == RCEC_MAGIC); 4476ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(cand_szB >= 1); 4477ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Count how many non-zero frames we have. */ 4478ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown maxNFrames = min_UInt(N_FRAMES, VG_(clo_backtrace_size)); 4479ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (n = 0; n < maxNFrames; n++) { 4480ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0 == cand_rcec->frames[n]) break; 4481ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4482b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov *resEC = VG_(make_ExeContext_from_StackTrace) 4483b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov (cand_rcec->frames, n); 4484b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov *resThr = Thr__from_ThrID(cand_thrid); 4485b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov *resSzB = cand_szB; 4486b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov *resIsW = cand_isW; 4487b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov *locksHeldW = cand_locksHeldW; 4488ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return True; 4489ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4490ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4491ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* consider next address in toCheck[] */ 4492ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } /* for (j = 0; j < nToCheck; j++) */ 4493ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4494ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* really didn't find anything. */ 4495ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 4496ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4497ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4498ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void event_map_init ( void ) 4499ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 4500ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i; 4501ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4502ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Context (RCEC) group allocator */ 4503ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown init_GroupAlloc ( &rcec_group_allocator, 4504ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sizeof(RCEC), 4505ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1000 /* RCECs per group */, 4506ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(zalloc), 4507ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "libhb.event_map_init.1 (RCEC groups)", 4508ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free) ); 4509ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4510ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Context table */ 4511ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(!contextTab); 4512ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown contextTab = HG_(zalloc)( "libhb.event_map_init.2 (context table)", 4513ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown N_RCEC_TAB * sizeof(RCEC*) ); 4514ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(contextTab); 4515ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_RCEC_TAB; i++) 4516ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown contextTab[i] = NULL; 4517ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4518ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Oldref group allocator */ 4519ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown init_GroupAlloc ( &oldref_group_allocator, 4520ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sizeof(OldRef), 4521ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1000 /* OldRefs per group */, 4522ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(zalloc), 4523ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "libhb.event_map_init.3 (OldRef groups)", 4524ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free) ); 4525ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4526ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Oldref tree */ 4527ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(!oldrefTree); 4528ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldrefTree = VG_(newSWA)( 4529ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(zalloc), 4530ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "libhb.event_map_init.4 (oldref tree)", 4531ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free) 4532ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ); 4533ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(oldrefTree); 4534ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4535ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldrefGen = 0; 4536ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldrefGenIncAt = 0; 4537ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldrefTreeN = 0; 4538ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4539ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4540ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void event_map__check_reference_counts ( Bool before ) 4541ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 4542ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown RCEC* rcec; 4543ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown OldRef* oldref; 4544ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i; 4545ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord nEnts = 0; 4546ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord keyW, valW; 4547ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4548ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Set the 'check' reference counts to zero. Also, optionally 4549ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown check that the real reference counts are non-zero. We allow 4550ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown these to fall to zero before a GC, but the GC must get rid of 4551ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown all those that are zero, hence none should be zero after a 4552ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown GC. */ 4553ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_RCEC_TAB; i++) { 4554ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (rcec = contextTab[i]; rcec; rcec = rcec->next) { 4555ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nEnts++; 4556ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(rcec); 4557ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(rcec->magic == RCEC_MAGIC); 4558ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!before) 4559ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(rcec->rc > 0); 4560ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcec->rcX = 0; 4561ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4562ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4563ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4564ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* check that the stats are sane */ 4565ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(nEnts == stats__ctxt_tab_curr); 4566ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(stats__ctxt_tab_curr <= stats__ctxt_tab_max); 4567ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4568ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* visit all the referencing points, inc check ref counts */ 4569ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(initIterSWA)( oldrefTree ); 4570ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (VG_(nextIterSWA)( oldrefTree, &keyW, &valW )) { 4571ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldref = (OldRef*)valW; 4572ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(oldref->magic == OldRef_MAGIC); 4573ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_OLDREF_ACCS; i++) { 4574b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID aThrID = oldref->accs[i].thrid; 4575b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov RCEC* aRef = oldref->accs[i].rcec; 4576b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (aThrID != 0) { 4577ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aRef); 4578ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aRef->magic == RCEC_MAGIC); 4579ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown aRef->rcX++; 4580ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 4581ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(!aRef); 4582ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4583ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4584ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4585ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4586ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* compare check ref counts with actual */ 4587ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_RCEC_TAB; i++) { 4588ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (rcec = contextTab[i]; rcec; rcec = rcec->next) { 4589ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(rcec->rc == rcec->rcX); 4590ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4591ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4592ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4593ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4594ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown__attribute__((noinline)) 4595ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void event_map_maybe_GC ( void ) 4596ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 4597ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown OldRef* oldref; 4598ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord keyW, valW, retained, maxGen; 4599ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown XArray* refs2del; 4600ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i, j, n2del; 4601ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4602ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord* genMap = NULL; 4603ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord genMap_min = 0; 4604ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord genMap_size = 0; 4605ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4606ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(oldrefTreeN < HG_(clo_conflict_cache_size))) 4607ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 4608ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4609ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) 4610ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("libhb: event_map GC at size %lu\n", oldrefTreeN); 4611ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4612ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Check for sane command line params. Limit values must match 4613ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown those in hg_process_cmd_line_option. */ 4614ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert( HG_(clo_conflict_cache_size) >= 10*1000 ); 4615ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert( HG_(clo_conflict_cache_size) <= 30*1000*1000 ); 4616ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4617ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Check our counting is sane (expensive) */ 4618ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_CEM) 4619ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(oldrefTreeN == VG_(sizeSWA)( oldrefTree )); 4620ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4621ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Check the reference counts (expensive) */ 4622ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_CEM) 4623ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown event_map__check_reference_counts( True/*before*/ ); 4624ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4625ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Compute the distribution of generation values in the ref tree. 4626ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown There are likely only to be a few different generation numbers 4627ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown in the whole tree, but we don't know what they are. Hence use a 4628ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown dynamically resized array of counters. The array is genMap[0 4629ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown .. genMap_size-1], where genMap[0] is the count for the 4630ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown generation number genMap_min, genMap[1] is the count for 4631ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown genMap_min+1, etc. If a new number is seen outside the range 4632ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown [genMap_min .. genMap_min + genMap_size - 1] then the array is 4633ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown copied into a larger array, and genMap_min and genMap_size are 4634ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown adjusted accordingly. */ 4635ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4636ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* genMap :: generation-number -> count-of-nodes-with-that-number */ 4637ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4638ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(initIterSWA)( oldrefTree ); 4639ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while ( VG_(nextIterSWA)( oldrefTree, &keyW, &valW )) { 4640ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4641ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord ea, key; 4642ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldref = (OldRef*)valW; 4643ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown key = oldref->gen; 4644ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4645ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* BEGIN find 'ea', which is the index in genMap holding the 4646ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown count for generation number 'key'. */ 4647ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(genMap == NULL)) { 4648ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* deal with the first key to be seen, so that the following 4649ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cases don't need to handle the complexity of a NULL count 4650ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown array. */ 4651ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown genMap_min = key; 4652ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown genMap_size = 1; 4653ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown genMap = HG_(zalloc)( "libhb.emmG.1a", 4654ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown genMap_size * sizeof(UWord) ); 4655ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ea = 0; 4656ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)("(%lu) case 1 [%lu .. %lu]\n", 4657ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown key, genMap_min, genMap_min+genMap_size- 1 ); 4658ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4659ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown else 4660ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(key >= genMap_min && key < genMap_min + genMap_size)) { 4661ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* this is the expected (almost-always-happens) case: 'key' 4662ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown is already mapped in the array. */ 4663ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ea = key - genMap_min; 4664ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4665ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown else 4666ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (key < genMap_min) { 4667ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* 'key' appears before the start of the current array. 4668ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Extend the current array by allocating a larger one and 4669ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown copying the current one to the upper end of it. */ 4670ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word more; 4671ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord* map2; 4672ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown more = genMap_min - key; 4673ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(more > 0); 4674ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown map2 = HG_(zalloc)( "libhb.emmG.1b", 4675ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (genMap_size + more) * sizeof(UWord) ); 4676ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(memcpy)( &map2[more], genMap, genMap_size * sizeof(UWord) ); 4677ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free)( genMap ); 4678ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown genMap = map2; 4679ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown genMap_size += more; 4680ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown genMap_min -= more; 4681ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ea = 0; 4682ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(genMap_min == key); 4683ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)("(%lu) case 2 [%lu .. %lu]\n", 4684ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown key, genMap_min, genMap_min+genMap_size- 1 ); 4685ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4686ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown else { 4687ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* 'key' appears after the end of the current array. Extend 4688ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown the current array by allocating a larger one and copying 4689ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown the current one to the lower end of it. */ 4690ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word more; 4691ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord* map2; 4692ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(key >= genMap_min + genMap_size); 4693ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown more = key - (genMap_min + genMap_size) + 1; 4694ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(more > 0); 4695ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown map2 = HG_(zalloc)( "libhb.emmG.1c", 4696ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (genMap_size + more) * sizeof(UWord) ); 4697ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(memcpy)( &map2[0], genMap, genMap_size * sizeof(UWord) ); 4698ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free)( genMap ); 4699ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown genMap = map2; 4700ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown genMap_size += more; 4701ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ea = genMap_size - 1;; 4702ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(genMap_min + genMap_size - 1 == key); 4703ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)("(%lu) case 3 [%lu .. %lu]\n", 4704ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown key, genMap_min, genMap_min+genMap_size- 1 ); 4705ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4706ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* END find 'ea' from 'key' */ 4707ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4708ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(ea >= 0 && ea < genMap_size); 4709ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* and the whole point of this elaborate computation of 'ea' is .. */ 4710ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown genMap[ea]++; 4711ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4712ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4713ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(genMap); 4714ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(genMap_size > 0); 4715ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4716ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Sanity check what we just computed */ 4717ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown { UWord sum = 0; 4718ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < genMap_size; i++) { 4719ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)(" xxx: gen %ld has %lu\n", 4720ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown i + genMap_min, genMap[i] ); 4721ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sum += genMap[i]; 4722ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4723ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sum == oldrefTreeN); 4724ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4725ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4726ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Figure out how many generations to throw away */ 4727ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown retained = oldrefTreeN; 4728ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown maxGen = 0; 4729ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4730ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < genMap_size; i++) { 4731ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown keyW = i + genMap_min; 4732ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown valW = genMap[i]; 4733ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(keyW > 0); /* can't allow a generation # 0 */ 4734ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)(" XXX: gen %lu has %lu\n", keyW, valW ); 4735ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(keyW >= maxGen); 4736ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(retained >= valW); 4737ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (retained - valW 4738ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown > (UWord)(HG_(clo_conflict_cache_size) 4739ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown * EVENT_MAP_GC_DISCARD_FRACTION)) { 4740ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown retained -= valW; 4741ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown maxGen = keyW; 4742ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 4743ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 4744ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4745ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4746ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4747ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free)(genMap); 4748ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4749ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(retained >= 0 && retained <= oldrefTreeN); 4750ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4751ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Now make up a big list of the oldrefTree entries we want to 4752ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown delete. We can't simultaneously traverse the tree and delete 4753ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stuff from it, so first we need to copy them off somewhere 4754ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown else. (sigh) */ 4755ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown refs2del = VG_(newXA)( HG_(zalloc), "libhb.emmG.2", 4756ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free), sizeof(Addr) ); 4757ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4758ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (retained < oldrefTreeN) { 4759ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4760ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* This is the normal (expected) case. We discard any ref whose 4761ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown generation number <= maxGen. */ 4762ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(initIterSWA)( oldrefTree ); 4763ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (VG_(nextIterSWA)( oldrefTree, &keyW, &valW )) { 4764ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldref = (OldRef*)valW; 4765ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(oldref->magic == OldRef_MAGIC); 4766ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (oldref->gen <= maxGen) { 4767ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(addToXA)( refs2del, &keyW ); 4768ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4769ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4770ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (VG_(clo_stats)) { 4771ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(message)(Vg_DebugMsg, 4772ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "libhb: EvM GC: delete generations %lu and below, " 4773ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "retaining %lu entries\n", 4774ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown maxGen, retained ); 4775ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4776ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4777ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 4778ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4779ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown static UInt rand_seed = 0; /* leave as static */ 4780ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4781ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Degenerate case: there's only one generation in the entire 4782ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tree, so we need to have some other way of deciding which 4783ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown refs to throw away. Just throw out half of them randomly. */ 4784ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(retained == oldrefTreeN); 4785ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(initIterSWA)( oldrefTree ); 4786ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (VG_(nextIterSWA)( oldrefTree, &keyW, &valW )) { 4787ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt n; 4788ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldref = (OldRef*)valW; 4789ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(oldref->magic == OldRef_MAGIC); 4790ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown n = VG_(random)( &rand_seed ); 4791ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if ((n & 0xFFF) < 0x800) { 4792ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(addToXA)( refs2del, &keyW ); 4793ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown retained--; 4794ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4795ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4796ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (VG_(clo_stats)) { 4797ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(message)(Vg_DebugMsg, 4798ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "libhb: EvM GC: randomly delete half the entries, " 4799ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "retaining %lu entries\n", 4800ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown retained ); 4801ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4802ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4803ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4804ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4805ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown n2del = VG_(sizeXA)( refs2del ); 4806ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(n2del == (Word)(oldrefTreeN - retained)); 4807ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4808ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)("%s","deleting entries\n"); 4809ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < n2del; i++) { 4810ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool b; 4811ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr ga2del = *(Addr*)VG_(indexXA)( refs2del, i ); 4812ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b = VG_(delFromSWA)( oldrefTree, &keyW, &valW, ga2del ); 4813ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(b); 4814ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(keyW == ga2del); 4815ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldref = (OldRef*)valW; 4816ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (j = 0; j < N_OLDREF_ACCS; j++) { 4817b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ThrID aThrID = oldref->accs[j].thrid; 4818b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov RCEC* aRef = oldref->accs[j].rcec; 4819ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (aRef) { 4820b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(aThrID != 0); 4821ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_rcdec3++; 4822ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ctxt__rcdec( aRef ); 4823ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 4824b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(aThrID == 0); 4825ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4826ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4827ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4828ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown free_OldRef( oldref ); 4829ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4830ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4831ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(deleteXA)( refs2del ); 4832ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4833ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert( VG_(sizeSWA)( oldrefTree ) == retained ); 4834ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4835ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldrefTreeN = retained; 4836ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown oldrefGenIncAt = oldrefTreeN; /* start new gen right away */ 4837ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4838ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Throw away all RCECs with zero reference counts */ 4839ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_RCEC_TAB; i++) { 4840ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown RCEC** pp = &contextTab[i]; 4841ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown RCEC* p = *pp; 4842ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (p) { 4843ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (p->rc == 0) { 4844ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *pp = p->next; 4845ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown free_RCEC(p); 4846ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown p = *pp; 4847ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(stats__ctxt_tab_curr > 0); 4848ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_tab_curr--; 4849ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 4850ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown pp = &p->next; 4851ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown p = p->next; 4852ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4853ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4854ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 4855ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4856ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Check the reference counts (expensive) */ 4857ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_CEM) 4858ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown event_map__check_reference_counts( False/*after*/ ); 4859ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4860ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //if (0) 4861ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //VG_(printf)("XXXX final sizes: oldrefTree %ld, contextTree %ld\n\n", 4862ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // VG_(OSetGen_Size)(oldrefTree), VG_(OSetGen_Size)(contextTree)); 4863ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4864ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 4865ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4866ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4867ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 4868ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 4869ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Core MSM // 4870ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 4871ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 4872ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4873ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Logic in msmcread/msmcwrite updated/verified after re-analysis, 19 4874ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Nov 08, and again after [...], 4875ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown June 09. */ 4876ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4877ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic ULong stats__msmcread = 0; 4878ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic ULong stats__msmcread_change = 0; 4879ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic ULong stats__msmcwrite = 0; 4880ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic ULong stats__msmcwrite_change = 0; 4881ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4882ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Some notes on the H1 history mechanism: 4883ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4884ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Transition rules are: 4885ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4886ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown read_{Kr,Kw}(Cr,Cw) = (Cr, Cr `join` Kw) 4887ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write_{Kr,Kw}(Cr,Cw) = (Cr `join` Kw, Cr `join` Kw) 4888ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4889ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown After any access by a thread T to a location L, L's constraint pair 4890ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Cr,Cw) has Cw[T] == T's Kw[T], that is, == T's scalar W-clock. 4891ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4892ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown After a race by thread T conflicting with some previous access by 4893ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown some other thread U, for a location with constraint (before 4894ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown processing the later access) (Cr,Cw), then Cw[U] is the segment in 4895ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown which the previously access lies. 4896ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4897ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Hence in record_race_info, we pass in Cfailed and Kfailed, which 4898ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown are compared so as to find out which thread(s) this access 4899ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown conflicts with. Once that is established, we also require the 4900ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown pre-update Cw for the location, so we can index into it for those 4901ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown threads, to get the scalar clock values for the point at which the 4902ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown former accesses were made. (In fact we only bother to do any of 4903ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown this for an arbitrarily chosen one of the conflicting threads, as 4904ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown that's simpler, it avoids flooding the user with vast amounts of 4905ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mostly useless information, and because the program is wrong if it 4906ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown contains any races at all -- so we don't really need to show all 4907ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown conflicting access pairs initially, so long as we only show none if 4908ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown none exist). 4909ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4910ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown --- 4911ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4912ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown That requires the auxiliary proof that 4913ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4914ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Cr `join` Kw)[T] == Kw[T] 4915ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4916ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Why should that be true? Because for any thread T, Kw[T] >= the 4917ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown scalar clock value for T known by any other thread. In other 4918ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown words, because T's value for its own scalar clock is at least as up 4919ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown to date as the value for it known by any other thread (that is true 4920ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for both the R- and W- scalar clocks). Hence no other thread will 4921ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown be able to feed in a value for that element (indirectly via a 4922ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown constraint) which will exceed Kw[T], and hence the join cannot 4923ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cause that particular element to advance. 4924ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 4925ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4926ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown__attribute__((noinline)) 4927ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void record_race_info ( Thr* acc_thr, 4928ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr acc_addr, SizeT szB, Bool isWrite, 4929ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID Cfailed, 4930ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID Kfailed, 4931ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID Cw ) 4932ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 4933ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Call here to report a race. We just hand it onwards to 4934ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(record_error_Race). If that in turn discovers that the 4935ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown error is going to be collected, then, at history_level 2, that 4936ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown queries the conflicting-event map. The alternative would be to 4937ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown query it right here. But that causes a lot of pointless queries 4938ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for errors which will shortly be discarded as duplicates, and 4939ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown can become a performance overhead; so we defer the query until 4940ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown we know the error is not a duplicate. */ 4941ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4942ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Stacks for the bounds of the (or one of the) conflicting 4943ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown segment(s). These are only set at history_level 1. */ 4944ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ExeContext* hist1_seg_start = NULL; 4945ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ExeContext* hist1_seg_end = NULL; 4946ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Thread* hist1_conf_thr = NULL; 4947ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4948ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(acc_thr); 4949b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(acc_thr->hgthread); 4950b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(acc_thr->hgthread->hbthr == acc_thr); 4951ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(HG_(clo_history_level) >= 0 && HG_(clo_history_level) <= 2); 4952ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4953ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (HG_(clo_history_level) == 1) { 4954ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool found; 4955ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word firstIx, lastIx; 4956ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ULong_n_EC key; 4957ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4958ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* At history_level 1, we must round up the relevant stack-pair 4959ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for the conflicting segment right now. This is because 4960ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown deferring it is complex; we can't (easily) put Kfailed and 4961ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Cfailed into the XError and wait for later without 4962ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown getting tied up in difficulties with VtsID reference 4963ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown counting. So just do it now. */ 4964ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Thr* confThr; 4965ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ULong confTym = 0; 4966ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Which thread are we in conflict with? There may be more than 4967ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown one, in which case VtsID__findFirst_notLEQ selects one arbitrarily 4968ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (in fact it's the one with the lowest Thr* value). */ 4969ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown confThr = VtsID__findFirst_notLEQ( Cfailed, Kfailed ); 4970ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* This must exist! since if it was NULL then there's no 4971ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown conflict (semantics of return value of 4972ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__findFirst_notLEQ), and msmc{read,write}, which has 4973ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown called us, just checked exactly this -- that there was in 4974ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown fact a race. */ 4975ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(confThr); 4976ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4977ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Get the scalar clock value that the conflicting thread 4978ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown introduced into the constraint. A careful examination of the 4979ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown base machine rules shows that this must be the same as the 4980ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown conflicting thread's scalar clock when it created this 4981ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown constraint. Hence we know the scalar clock of the 4982ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown conflicting thread when the conflicting access was made. */ 4983ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown confTym = VtsID__indexAt( Cfailed, confThr ); 4984ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4985ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Using this scalar clock, index into the conflicting thread's 4986ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown collection of stack traces made each time its vector clock 4987ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (hence its scalar clock) changed. This gives the stack 4988ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown traces at the start and end of the conflicting segment (well, 4989ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown as per comment just above, of one of the conflicting 4990ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown segments, if there are more than one). */ 4991ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown key.ull = confTym; 4992ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown key.ec = NULL; 4993ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* tl_assert(confThr); -- asserted just above */ 4994ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(confThr->local_Kws_n_stacks); 4995ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown firstIx = lastIx = 0; 4996ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown found = VG_(lookupXA_UNSAFE)( 4997ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown confThr->local_Kws_n_stacks, 4998ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown &key, &firstIx, &lastIx, 4999ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int(*)(void*,void*))cmp__ULong_n_EC__by_ULong 5000ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ); 5001ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)("record_race_info %u %u %u confThr %p " 5002ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "confTym %llu found %d (%lu,%lu)\n", 5003ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Cfailed, Kfailed, Cw, 5004ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown confThr, confTym, found, firstIx, lastIx); 5005ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* We can't indefinitely collect stack traces at VTS 5006ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown transitions, since we'd eventually run out of memory. Hence 5007ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown note_local_Kw_n_stack_for will eventually throw away old 5008ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ones, which in turn means we might fail to find index value 5009ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown confTym in the array. */ 5010ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (found) { 5011ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ULong_n_EC *pair_start, *pair_end; 5012ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown pair_start 5013ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown = (ULong_n_EC*)VG_(indexXA)( confThr->local_Kws_n_stacks, lastIx ); 5014ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown hist1_seg_start = pair_start->ec; 5015ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (lastIx+1 < VG_(sizeXA)( confThr->local_Kws_n_stacks )) { 5016ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown pair_end 5017ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown = (ULong_n_EC*)VG_(indexXA)( confThr->local_Kws_n_stacks, 5018ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lastIx+1 ); 5019ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* from properties of VG_(lookupXA) and the comparison fn used: */ 5020ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(pair_start->ull < pair_end->ull); 5021ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown hist1_seg_end = pair_end->ec; 5022ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Could do a bit better here. It may be that pair_end 5023ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown doesn't have a stack, but the following entries in the 5024ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown array have the same scalar Kw and to have a stack. So 5025ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown we should search a bit further along the array than 5026ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lastIx+1 if hist1_seg_end is NULL. */ 5027ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 5028b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (!confThr->llexit_done) 5029ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown hist1_seg_end = main_get_EC( confThr ); 5030ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5031ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // seg_start could be NULL iff this is the first stack in the thread 5032ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //if (seg_start) VG_(pp_ExeContext)(seg_start); 5033ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //if (seg_end) VG_(pp_ExeContext)(seg_end); 5034b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov hist1_conf_thr = confThr->hgthread; 5035ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5036ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5037ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5038b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov HG_(record_error_Race)( acc_thr->hgthread, acc_addr, 5039ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown szB, isWrite, 5040ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown hist1_conf_thr, hist1_seg_start, hist1_seg_end ); 5041ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5042ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5043ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Bool is_sane_SVal_C ( SVal sv ) { 5044ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool leq; 5045ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!SVal__isC(sv)) return True; 5046ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown leq = VtsID__cmpLEQ( SVal__unC_Rmin(sv), SVal__unC_Wmin(sv) ); 5047ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return leq; 5048ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5049ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5050ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5051ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Compute new state following a read */ 5052ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline SVal msmcread ( SVal svOld, 5053ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* The following are only needed for 5054ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown creating error reports. */ 5055ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Thr* acc_thr, 5056ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr acc_addr, SizeT szB ) 5057ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 5058ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal svNew = SVal_INVALID; 5059ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__msmcread++; 5060ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5061ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Redundant sanity check on the constraints */ 5062ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_MSM) { 5063ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_SVal_C(svOld)); 5064ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5065ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5066ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(SVal__isC(svOld))) { 5067ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID tviR = acc_thr->viR; 5068ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID tviW = acc_thr->viW; 5069ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID rmini = SVal__unC_Rmin(svOld); 5070ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID wmini = SVal__unC_Wmin(svOld); 5071ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool leq = VtsID__cmpLEQ(rmini,tviR); 5072ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(leq)) { 5073ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* no race */ 5074ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Note: RWLOCK subtlety: use tviW, not tviR */ 5075ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = SVal__mkC( rmini, VtsID__join2(wmini, tviW) ); 5076ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto out; 5077ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 5078ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* assert on sanity of constraints. */ 5079ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool leqxx = VtsID__cmpLEQ(rmini,wmini); 5080ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(leqxx); 5081ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // same as in non-race case 5082ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = SVal__mkC( rmini, VtsID__join2(wmini, tviW) ); 5083ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown record_race_info( acc_thr, acc_addr, szB, False/*!isWrite*/, 5084ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rmini, /* Cfailed */ 5085ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tviR, /* Kfailed */ 5086ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown wmini /* Cw */ ); 5087ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto out; 5088ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5089ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5090ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (SVal__isA(svOld)) { 5091ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* reading no-access memory (sigh); leave unchanged */ 5092ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* check for no pollution */ 5093ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svOld == SVal_NOACCESS); 5094ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = SVal_NOACCESS; 5095ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto out; 5096ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5097ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)("msmcread: bad svOld: 0x%016llx\n", svOld); 5098ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 5099ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5100ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown out: 5101ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_MSM) { 5102ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_SVal_C(svNew)); 5103ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5104ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(svNew != svOld)) { 5105ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5106ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (HG_(clo_history_level) >= 2 5107ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown && SVal__isC(svOld) && SVal__isC(svNew)) { 5108ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown event_map_bind( acc_addr, szB, False/*!isWrite*/, acc_thr ); 5109ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__msmcread_change++; 5110ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5111ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5112ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return svNew; 5113ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5114ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5115ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5116ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Compute new state following a write */ 5117ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline SVal msmcwrite ( SVal svOld, 5118ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* The following are only needed for 5119ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown creating error reports. */ 5120ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Thr* acc_thr, 5121ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr acc_addr, SizeT szB ) 5122ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 5123ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal svNew = SVal_INVALID; 5124ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__msmcwrite++; 5125ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5126ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Redundant sanity check on the constraints */ 5127ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_MSM) { 5128ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_SVal_C(svOld)); 5129ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5130ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5131ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(SVal__isC(svOld))) { 5132ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID tviW = acc_thr->viW; 5133ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID wmini = SVal__unC_Wmin(svOld); 5134ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool leq = VtsID__cmpLEQ(wmini,tviW); 5135ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(leq)) { 5136ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* no race */ 5137ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = SVal__mkC( tviW, tviW ); 5138ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto out; 5139ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 5140ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID rmini = SVal__unC_Rmin(svOld); 5141ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* assert on sanity of constraints. */ 5142ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool leqxx = VtsID__cmpLEQ(rmini,wmini); 5143ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(leqxx); 5144ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // same as in non-race case 5145ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // proof: in the non-race case, we have 5146ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // rmini <= wmini (invar on constraints) 5147ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // tviW <= tviR (invar on thread clocks) 5148ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // wmini <= tviW (from run-time check) 5149ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // hence from transitivity of <= we have 5150ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // rmini <= wmini <= tviW 5151ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // and so join(rmini,tviW) == tviW 5152ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // and join(wmini,tviW) == tviW 5153ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // qed. 5154ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = SVal__mkC( VtsID__join2(rmini, tviW), 5155ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__join2(wmini, tviW) ); 5156ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown record_race_info( acc_thr, acc_addr, szB, True/*isWrite*/, 5157ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown wmini, /* Cfailed */ 5158ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tviW, /* Kfailed */ 5159ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown wmini /* Cw */ ); 5160ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto out; 5161ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5162ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5163ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (SVal__isA(svOld)) { 5164ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* writing no-access memory (sigh); leave unchanged */ 5165ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* check for no pollution */ 5166ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svOld == SVal_NOACCESS); 5167ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = SVal_NOACCESS; 5168ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto out; 5169ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5170ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)("msmcwrite: bad svOld: 0x%016llx\n", svOld); 5171ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 5172ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5173ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown out: 5174ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_MSM) { 5175ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_SVal_C(svNew)); 5176ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5177ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(svNew != svOld)) { 5178ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5179ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (HG_(clo_history_level) >= 2 5180ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown && SVal__isC(svOld) && SVal__isC(svNew)) { 5181ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown event_map_bind( acc_addr, szB, True/*isWrite*/, acc_thr ); 5182ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__msmcwrite_change++; 5183ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5184ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5185ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return svNew; 5186ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5187ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5188ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5189ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 5190ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 5191ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Apply core MSM to specific memory locations // 5192ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 5193ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 5194ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5195ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------- ZSM accesses: 8 bit sapply ------------- */ 5196ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5197ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_sapply08__msmcread ( Thr* thr, Addr a ) { 5198ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 5199ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord cloff, tno, toff; 5200ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal svOld, svNew; 5201ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr; 5202ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cread08s++; 5203ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = get_cacheline(a); 5204ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cloff = get_cacheline_offset(a); 5205ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tno = get_treeno(a); 5206ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown toff = get_tree_offset(a); /* == 0 .. 7 */ 5207ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = cl->descrs[tno]; 5208ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !(descr & (TREE_DESCR_8_0 << toff)) )) { 5209ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal* tree = &cl->svals[tno << 3]; 5210ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = pulldown_to_8(tree, toff, descr); 5211ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5212ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */ 5213ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5214ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svOld = cl->svals[cloff]; 5215ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = msmcread( svOld, thr,a,1 ); 5216ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5217ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5218ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff] = svNew; 5219ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5220ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5221ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_sapply08__msmcwrite ( Thr* thr, Addr a ) { 5222ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 5223ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord cloff, tno, toff; 5224ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal svOld, svNew; 5225ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr; 5226ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cwrite08s++; 5227ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = get_cacheline(a); 5228ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cloff = get_cacheline_offset(a); 5229ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tno = get_treeno(a); 5230ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown toff = get_tree_offset(a); /* == 0 .. 7 */ 5231ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = cl->descrs[tno]; 5232ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !(descr & (TREE_DESCR_8_0 << toff)) )) { 5233ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal* tree = &cl->svals[tno << 3]; 5234ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = pulldown_to_8(tree, toff, descr); 5235ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5236ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */ 5237ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5238ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svOld = cl->svals[cloff]; 5239ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = msmcwrite( svOld, thr,a,1 ); 5240ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5241ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5242ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff] = svNew; 5243ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5244ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5245ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------- ZSM accesses: 16 bit sapply ------------- */ 5246ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5247ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_sapply16__msmcread ( Thr* thr, Addr a ) { 5248ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 5249ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord cloff, tno, toff; 5250ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal svOld, svNew; 5251ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr; 5252ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cread16s++; 5253ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(!aligned16(a))) goto slowcase; 5254ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = get_cacheline(a); 5255ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cloff = get_cacheline_offset(a); 5256ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tno = get_treeno(a); 5257ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown toff = get_tree_offset(a); /* == 0, 2, 4 or 6 */ 5258ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = cl->descrs[tno]; 5259ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !(descr & (TREE_DESCR_16_0 << toff)) )) { 5260ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (valid_value_is_below_me_16(descr, toff)) { 5261ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto slowcase; 5262ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 5263ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal* tree = &cl->svals[tno << 3]; 5264ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = pulldown_to_16(tree, toff, descr); 5265ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5266ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5267ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */ 5268ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5269ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svOld = cl->svals[cloff]; 5270ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = msmcread( svOld, thr,a,2 ); 5271ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5272ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5273ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff] = svNew; 5274ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5275ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown slowcase: /* misaligned, or must go further down the tree */ 5276ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_16to8splits++; 5277ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply08__msmcread( thr, a + 0 ); 5278ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply08__msmcread( thr, a + 1 ); 5279ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5280ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5281ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_sapply16__msmcwrite ( Thr* thr, Addr a ) { 5282ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 5283ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord cloff, tno, toff; 5284ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal svOld, svNew; 5285ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr; 5286ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cwrite16s++; 5287ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(!aligned16(a))) goto slowcase; 5288ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = get_cacheline(a); 5289ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cloff = get_cacheline_offset(a); 5290ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tno = get_treeno(a); 5291ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown toff = get_tree_offset(a); /* == 0, 2, 4 or 6 */ 5292ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = cl->descrs[tno]; 5293ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !(descr & (TREE_DESCR_16_0 << toff)) )) { 5294ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (valid_value_is_below_me_16(descr, toff)) { 5295ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto slowcase; 5296ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 5297ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal* tree = &cl->svals[tno << 3]; 5298ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = pulldown_to_16(tree, toff, descr); 5299ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5300ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5301ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */ 5302ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5303ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svOld = cl->svals[cloff]; 5304ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = msmcwrite( svOld, thr,a,2 ); 5305ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5306ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5307ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff] = svNew; 5308ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5309ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown slowcase: /* misaligned, or must go further down the tree */ 5310ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_16to8splits++; 5311ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply08__msmcwrite( thr, a + 0 ); 5312ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply08__msmcwrite( thr, a + 1 ); 5313ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5314ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5315ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------- ZSM accesses: 32 bit sapply ------------- */ 5316ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5317ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_sapply32__msmcread ( Thr* thr, Addr a ) { 5318ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 5319ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord cloff, tno, toff; 5320ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal svOld, svNew; 5321ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr; 5322ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cread32s++; 5323ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(!aligned32(a))) goto slowcase; 5324ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = get_cacheline(a); 5325ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cloff = get_cacheline_offset(a); 5326ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tno = get_treeno(a); 5327ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown toff = get_tree_offset(a); /* == 0 or 4 */ 5328ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = cl->descrs[tno]; 5329ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !(descr & (TREE_DESCR_32_0 << toff)) )) { 5330ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (valid_value_is_above_me_32(descr, toff)) { 5331ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal* tree = &cl->svals[tno << 3]; 5332ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = pulldown_to_32(tree, toff, descr); 5333ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 5334ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto slowcase; 5335ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5336ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5337ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */ 5338ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5339ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svOld = cl->svals[cloff]; 5340ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = msmcread( svOld, thr,a,4 ); 5341ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5342ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5343ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff] = svNew; 5344ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5345ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown slowcase: /* misaligned, or must go further down the tree */ 5346ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_32to16splits++; 5347ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply16__msmcread( thr, a + 0 ); 5348ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply16__msmcread( thr, a + 2 ); 5349ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5350ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5351ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_sapply32__msmcwrite ( Thr* thr, Addr a ) { 5352ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 5353ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord cloff, tno, toff; 5354ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal svOld, svNew; 5355ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr; 5356ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cwrite32s++; 5357ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(!aligned32(a))) goto slowcase; 5358ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = get_cacheline(a); 5359ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cloff = get_cacheline_offset(a); 5360ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tno = get_treeno(a); 5361ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown toff = get_tree_offset(a); /* == 0 or 4 */ 5362ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = cl->descrs[tno]; 5363ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !(descr & (TREE_DESCR_32_0 << toff)) )) { 5364ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (valid_value_is_above_me_32(descr, toff)) { 5365ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal* tree = &cl->svals[tno << 3]; 5366ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = pulldown_to_32(tree, toff, descr); 5367ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 5368ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto slowcase; 5369ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5370ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5371ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */ 5372ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5373ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svOld = cl->svals[cloff]; 5374ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = msmcwrite( svOld, thr,a,4 ); 5375ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5376ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5377ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff] = svNew; 5378ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5379ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown slowcase: /* misaligned, or must go further down the tree */ 5380ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_32to16splits++; 5381ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply16__msmcwrite( thr, a + 0 ); 5382ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply16__msmcwrite( thr, a + 2 ); 5383ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5384ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5385ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------- ZSM accesses: 64 bit sapply ------------- */ 5386ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5387ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_sapply64__msmcread ( Thr* thr, Addr a ) { 5388ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 5389ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord cloff, tno; 5390ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //UWord toff; 5391ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal svOld, svNew; 5392ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr; 5393ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cread64s++; 5394ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(!aligned64(a))) goto slowcase; 5395ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = get_cacheline(a); 5396ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cloff = get_cacheline_offset(a); 5397ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tno = get_treeno(a); 5398ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //toff = get_tree_offset(a); /* == 0, unused */ 5399ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = cl->descrs[tno]; 5400ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !(descr & TREE_DESCR_64) )) { 5401ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto slowcase; 5402ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5403ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svOld = cl->svals[cloff]; 5404ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = msmcread( svOld, thr,a,8 ); 5405ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5406ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5407ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff] = svNew; 5408ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5409ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown slowcase: /* misaligned, or must go further down the tree */ 5410ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_64to32splits++; 5411ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply32__msmcread( thr, a + 0 ); 5412ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply32__msmcread( thr, a + 4 ); 5413ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5414ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5415ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_sapply64__msmcwrite ( Thr* thr, Addr a ) { 5416ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 5417ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord cloff, tno; 5418ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //UWord toff; 5419ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal svOld, svNew; 5420ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr; 5421ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cwrite64s++; 5422ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(!aligned64(a))) goto slowcase; 5423ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = get_cacheline(a); 5424ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cloff = get_cacheline_offset(a); 5425ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tno = get_treeno(a); 5426ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //toff = get_tree_offset(a); /* == 0, unused */ 5427ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = cl->descrs[tno]; 5428ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !(descr & TREE_DESCR_64) )) { 5429ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown goto slowcase; 5430ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5431ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svOld = cl->svals[cloff]; 5432ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown svNew = msmcwrite( svOld, thr,a,8 ); 5433ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5434ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5435ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff] = svNew; 5436ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5437ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown slowcase: /* misaligned, or must go further down the tree */ 5438ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_64to32splits++; 5439ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply32__msmcwrite( thr, a + 0 ); 5440ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply32__msmcwrite( thr, a + 4 ); 5441ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5442ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5443ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--------------- ZSM accesses: 8 bit swrite --------------- */ 5444ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5445ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 5446ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_swrite08 ( Addr a, SVal svNew ) { 5447ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 5448ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord cloff, tno, toff; 5449ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr; 5450ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_swrite08s++; 5451ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = get_cacheline(a); 5452ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cloff = get_cacheline_offset(a); 5453ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tno = get_treeno(a); 5454ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown toff = get_tree_offset(a); /* == 0 .. 7 */ 5455ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = cl->descrs[tno]; 5456ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !(descr & (TREE_DESCR_8_0 << toff)) )) { 5457ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal* tree = &cl->svals[tno << 3]; 5458ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = pulldown_to_8(tree, toff, descr); 5459ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5460ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */ 5461ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5462ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5463ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff] = svNew; 5464ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5465ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5466ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--------------- ZSM accesses: 16 bit swrite --------------- */ 5467ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5468ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 5469ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_swrite16 ( Addr a, SVal svNew ) { 5470ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 5471ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord cloff, tno, toff; 5472ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr; 5473ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_swrite16s++; 5474ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(!aligned16(a))) goto slowcase; 5475ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = get_cacheline(a); 5476ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cloff = get_cacheline_offset(a); 5477ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tno = get_treeno(a); 5478ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown toff = get_tree_offset(a); /* == 0, 2, 4 or 6 */ 5479ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = cl->descrs[tno]; 5480ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !(descr & (TREE_DESCR_16_0 << toff)) )) { 5481ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (valid_value_is_below_me_16(descr, toff)) { 5482ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Writing at this level. Need to fix up 'descr'. */ 5483ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = pullup_descr_to_16(descr, toff); 5484ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* At this point, the tree does not match cl->descr[tno] any 5485ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown more. The assignments below will fix it up. */ 5486ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 5487ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* We can't indiscriminately write on the w16 node as in the 5488ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown w64 case, as that might make the node inconsistent with 5489ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown its parent. So first, pull down to this level. */ 5490ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal* tree = &cl->svals[tno << 3]; 5491ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = pulldown_to_16(tree, toff, descr); 5492ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5493ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */ 5494ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5495ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5496ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5497ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 0] = svNew; 5498ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 1] = SVal_INVALID; 5499ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5500ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown slowcase: /* misaligned */ 5501ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_16to8splits++; 5502ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite08( a + 0, svNew ); 5503ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite08( a + 1, svNew ); 5504ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5505ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5506ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--------------- ZSM accesses: 32 bit swrite --------------- */ 5507ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5508ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 5509ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_swrite32 ( Addr a, SVal svNew ) { 5510ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 5511ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord cloff, tno, toff; 5512ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr; 5513ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_swrite32s++; 5514ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(!aligned32(a))) goto slowcase; 5515ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = get_cacheline(a); 5516ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cloff = get_cacheline_offset(a); 5517ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tno = get_treeno(a); 5518ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown toff = get_tree_offset(a); /* == 0 or 4 */ 5519ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = cl->descrs[tno]; 5520ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !(descr & (TREE_DESCR_32_0 << toff)) )) { 5521ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (valid_value_is_above_me_32(descr, toff)) { 5522ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* We can't indiscriminately write on the w32 node as in the 5523ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown w64 case, as that might make the node inconsistent with 5524ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown its parent. So first, pull down to this level. */ 5525ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal* tree = &cl->svals[tno << 3]; 5526ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = pulldown_to_32(tree, toff, descr); 5527ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (CHECK_ZSM) 5528ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_CacheLine(cl)); /* EXPENSIVE */ 5529ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 5530ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Writing at this level. Need to fix up 'descr'. */ 5531ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = pullup_descr_to_32(descr, toff); 5532ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* At this point, the tree does not match cl->descr[tno] any 5533ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown more. The assignments below will fix it up. */ 5534ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5535ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5536ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5537ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 0] = svNew; 5538ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 1] = SVal_INVALID; 5539ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 2] = SVal_INVALID; 5540ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 3] = SVal_INVALID; 5541ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5542ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown slowcase: /* misaligned */ 5543ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_32to16splits++; 5544ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite16( a + 0, svNew ); 5545ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite16( a + 2, svNew ); 5546ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5547ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5548ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--------------- ZSM accesses: 64 bit swrite --------------- */ 5549ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5550ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 5551ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_swrite64 ( Addr a, SVal svNew ) { 5552ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 5553ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord cloff, tno; 5554ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //UWord toff; 5555ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_swrite64s++; 5556ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(!aligned64(a))) goto slowcase; 5557ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = get_cacheline(a); 5558ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cloff = get_cacheline_offset(a); 5559ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tno = get_treeno(a); 5560ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //toff = get_tree_offset(a); /* == 0, unused */ 5561ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = TREE_DESCR_64; 5562ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5563ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 0] = svNew; 5564ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 1] = SVal_INVALID; 5565ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 2] = SVal_INVALID; 5566ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 3] = SVal_INVALID; 5567ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 4] = SVal_INVALID; 5568ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 5] = SVal_INVALID; 5569ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 6] = SVal_INVALID; 5570ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->svals[cloff + 7] = SVal_INVALID; 5571ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5572ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown slowcase: /* misaligned */ 5573ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_64to32splits++; 5574ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite32( a + 0, svNew ); 5575ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite32( a + 4, svNew ); 5576ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5577ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5578ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------- ZSM accesses: 8 bit sread/scopy ------------- */ 5579ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5580ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 5581ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownSVal zsm_sread08 ( Addr a ) { 5582ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown CacheLine* cl; 5583ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord cloff, tno, toff; 5584ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UShort descr; 5585ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_sread08s++; 5586ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl = get_cacheline(a); 5587ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cloff = get_cacheline_offset(a); 5588ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tno = get_treeno(a); 5589ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown toff = get_tree_offset(a); /* == 0 .. 7 */ 5590ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown descr = cl->descrs[tno]; 5591ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY( !(descr & (TREE_DESCR_8_0 << toff)) )) { 5592ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal* tree = &cl->svals[tno << 3]; 5593ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cl->descrs[tno] = pulldown_to_8(tree, toff, descr); 5594ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5595ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return cl->svals[cloff]; 5596ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5597ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5598ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_scopy08 ( Addr src, Addr dst, Bool uu_normalise ) { 5599ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal sv; 5600ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_scopy08s++; 5601ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sv = zsm_sread08( src ); 5602ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite08( dst, sv ); 5603ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5604ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5605ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5606ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Block-copy states (needed for implementing realloc()). Note this 5607ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown doesn't change the filtering arrangements. The caller of 5608ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_scopy_range needs to attend to that. */ 5609ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5610ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_scopy_range ( Addr src, Addr dst, SizeT len ) 5611ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 5612ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT i; 5613ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) 5614ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5615ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5616ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* assert for non-overlappingness */ 5617ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(src+len <= dst || dst+len <= src); 5618ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5619ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* To be simple, just copy byte by byte. But so as not to wreck 5620ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown performance for later accesses to dst[0 .. len-1], normalise 5621ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown destination lines as we finish with them, and also normalise the 5622ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown line containing the first and last address. */ 5623ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < len; i++) { 5624ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool normalise 5625ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown = get_cacheline_offset( dst+i+1 ) == 0 /* last in line */ 5626ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown || i == 0 /* first in range */ 5627ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown || i == len-1; /* last in range */ 5628ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_scopy08( src+i, dst+i, normalise ); 5629ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5630ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5631ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5632ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5633ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* For setting address ranges to a given value. Has considerable 5634ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sophistication so as to avoid generating large numbers of pointless 5635ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cache loads/writebacks for large ranges. */ 5636ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5637ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Do small ranges in-cache, in the obvious way. */ 5638ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 5639ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_sset_range_SMALL ( Addr a, SizeT len, SVal svNew ) 5640ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 5641ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* fast track a couple of common cases */ 5642ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 4 && aligned32(a)) { 5643ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite32( a, svNew ); 5644ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5645ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5646ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 8 && aligned64(a)) { 5647ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite64( a, svNew ); 5648ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5649ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5650ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5651ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* be completely general (but as efficient as possible) */ 5652ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5653ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5654ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!aligned16(a) && len >= 1) { 5655ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite08( a, svNew ); 5656ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 1; 5657ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 1; 5658ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned16(a)); 5659ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5660ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5661ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5662ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!aligned32(a) && len >= 2) { 5663ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite16( a, svNew ); 5664ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 2; 5665ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 2; 5666ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned32(a)); 5667ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5668ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5669ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5670ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!aligned64(a) && len >= 4) { 5671ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite32( a, svNew ); 5672ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 4; 5673ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 4; 5674ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned64(a)); 5675ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5676ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5677ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5678ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 8) { 5679ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned64(a)); 5680ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (len >= 8) { 5681ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite64( a, svNew ); 5682ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 8; 5683ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 8; 5684ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5685ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned64(a)); 5686ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5687ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5688ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5689ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 4) 5690ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned32(a)); 5691ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 4) { 5692ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite32( a, svNew ); 5693ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 4; 5694ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 4; 5695ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5696ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5697ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5698ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 2) 5699ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned16(a)); 5700ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 2) { 5701ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite16( a, svNew ); 5702ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 2; 5703ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 2; 5704ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5705ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5706ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5707ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 1) { 5708ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite08( a, svNew ); 5709ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //a += 1; 5710ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 1; 5711ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5712ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(len == 0); 5713ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5714ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5715ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5716ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* If we're doing a small range, hand off to zsm_sset_range_SMALL. But 5717ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for larger ranges, try to operate directly on the out-of-cache 5718ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown representation, rather than dragging lines into the cache, 5719ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown overwriting them, and forcing them out. This turns out to be an 5720ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown important performance optimisation. 5721ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5722ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Note that this doesn't change the filtering arrangements. The 5723ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown caller of zsm_sset_range needs to attend to that. */ 5724ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5725ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void zsm_sset_range ( Addr a, SizeT len, SVal svNew ) 5726ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 5727ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(svNew != SVal_INVALID); 5728ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_make_New_arange += (ULong)len; 5729ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5730ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0 && len > 500) 5731ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("make New ( %#lx, %ld )\n", a, len ); 5732ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5733ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) { 5734ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown static UWord n_New_in_cache = 0; 5735ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown static UWord n_New_not_in_cache = 0; 5736ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* tag is 'a' with the in-line offset masked out, 5737ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown eg a[31]..a[4] 0000 */ 5738ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr tag = a & ~(N_LINE_ARANGE - 1); 5739ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord wix = (a >> N_LINE_BITS) & (N_WAY_NENT - 1); 5740ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(tag == cache_shmem.tags0[wix])) { 5741ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown n_New_in_cache++; 5742ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 5743ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown n_New_not_in_cache++; 5744ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5745ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0 == ((n_New_in_cache + n_New_not_in_cache) % 100000)) 5746ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("shadow_mem_make_New: IN %lu OUT %lu\n", 5747ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown n_New_in_cache, n_New_not_in_cache ); 5748ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5749ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5750ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(len < 2 * N_LINE_ARANGE)) { 5751ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sset_range_SMALL( a, len, svNew ); 5752ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 5753ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr before_start = a; 5754ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr aligned_start = cacheline_ROUNDUP(a); 5755ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr after_start = cacheline_ROUNDDN(a + len); 5756ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord before_len = aligned_start - before_start; 5757ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord aligned_len = after_start - aligned_start; 5758ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord after_len = a + len - after_start; 5759ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(before_start <= aligned_start); 5760ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned_start <= after_start); 5761ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(before_len < N_LINE_ARANGE); 5762ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(after_len < N_LINE_ARANGE); 5763ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(get_cacheline_offset(aligned_start) == 0); 5764ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (get_cacheline_offset(a) == 0) { 5765ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(before_len == 0); 5766ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(a == aligned_start); 5767ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5768ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (get_cacheline_offset(a+len) == 0) { 5769ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(after_len == 0); 5770ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(after_start == a+len); 5771ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5772ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (before_len > 0) { 5773ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sset_range_SMALL( before_start, before_len, svNew ); 5774ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5775ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (after_len > 0) { 5776ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sset_range_SMALL( after_start, after_len, svNew ); 5777ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5778ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_make_New_inZrep += (ULong)aligned_len; 5779ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5780ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (1) { 5781ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr tag; 5782ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord wix; 5783ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (aligned_start >= after_start) 5784ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 5785ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(get_cacheline_offset(aligned_start) == 0); 5786ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tag = aligned_start & ~(N_LINE_ARANGE - 1); 5787ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown wix = (aligned_start >> N_LINE_BITS) & (N_WAY_NENT - 1); 5788ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (tag == cache_shmem.tags0[wix]) { 5789ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i; 5790ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_LINE_ARANGE / 8; i++) 5791ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_swrite64( aligned_start + i * 8, svNew ); 5792ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 5793ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i; 5794ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word zix; 5795ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SecMap* sm; 5796ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown LineZ* lineZ; 5797ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* This line is not in the cache. Do not force it in; instead 5798ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown modify it in-place. */ 5799ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* find the Z line to write in and rcdec it or the 5800ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown associated F line. */ 5801ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown find_Z_for_writing( &sm, &zix, tag ); 5802ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(sm); 5803ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(zix >= 0 && zix < N_SECMAP_ZLINES); 5804ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineZ = &sm->linesZ[zix]; 5805ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineZ->dict[0] = svNew; 5806ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineZ->dict[1] = lineZ->dict[2] = lineZ->dict[3] = SVal_INVALID; 5807ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_LINE_ARANGE/4; i++) 5808ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lineZ->ix2s[i] = 0; /* all refer to dict[0] */ 5809ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown rcinc_LineZ(lineZ); 5810ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5811ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown aligned_start += N_LINE_ARANGE; 5812ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown aligned_len -= N_LINE_ARANGE; 5813ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5814ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned_start == after_start); 5815ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned_len == 0); 5816ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5817ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5818ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5819ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5820ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 5821ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 5822ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Front-filtering accesses // 5823ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 5824ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 5825ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5826ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__f_ac = 0; 5827ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic UWord stats__f_sk = 0; 5828ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5829ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#if 0 5830ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# define STATS__F_SHOW \ 5831ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown do { \ 5832ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (UNLIKELY(0 == (stats__f_ac & 0xFFFFFF))) \ 5833ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("filters: ac %lu sk %lu\n", \ 5834ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_ac, stats__f_sk); \ 5835ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } while (0) 5836ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#else 5837ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# define STATS__F_SHOW /* */ 5838ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#endif 5839ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5840ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_sapply08_f__msmcwrite ( Thr* thr, Addr a ) { 5841ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_ac++; 5842ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown STATS__F_SHOW; 5843ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(Filter__ok_to_skip_cwr08(thr->filter, a))) { 5844ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_sk++; 5845ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5846ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5847ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply08__msmcwrite(thr, a); 5848ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5849ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5850ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_sapply16_f__msmcwrite ( Thr* thr, Addr a ) { 5851ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_ac++; 5852ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown STATS__F_SHOW; 5853ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(Filter__ok_to_skip_cwr16(thr->filter, a))) { 5854ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_sk++; 5855ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5856ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5857ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply16__msmcwrite(thr, a); 5858ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5859ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5860ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_sapply32_f__msmcwrite ( Thr* thr, Addr a ) { 5861ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_ac++; 5862ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown STATS__F_SHOW; 5863ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(Filter__ok_to_skip_cwr32(thr->filter, a))) { 5864ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_sk++; 5865ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5866ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5867ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply32__msmcwrite(thr, a); 5868ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5869ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5870ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_sapply64_f__msmcwrite ( Thr* thr, Addr a ) { 5871ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_ac++; 5872ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown STATS__F_SHOW; 5873ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(Filter__ok_to_skip_cwr64(thr->filter, a))) { 5874ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_sk++; 5875ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5876ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5877ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply64__msmcwrite(thr, a); 5878ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5879ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5880ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_sapplyNN_f__msmcwrite ( Thr* thr, Addr a, SizeT len ) 5881ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 5882ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* fast track a couple of common cases */ 5883ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 4 && aligned32(a)) { 5884ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply32_f__msmcwrite( thr, a ); 5885ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5886ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5887ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 8 && aligned64(a)) { 5888ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply64_f__msmcwrite( thr, a ); 5889ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5890ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5891ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5892ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* be completely general (but as efficient as possible) */ 5893ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5894ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5895ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!aligned16(a) && len >= 1) { 5896ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply08_f__msmcwrite( thr, a ); 5897ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 1; 5898ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 1; 5899ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned16(a)); 5900ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5901ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5902ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5903ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!aligned32(a) && len >= 2) { 5904ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply16_f__msmcwrite( thr, a ); 5905ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 2; 5906ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 2; 5907ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned32(a)); 5908ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5909ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5910ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5911ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!aligned64(a) && len >= 4) { 5912ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply32_f__msmcwrite( thr, a ); 5913ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 4; 5914ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 4; 5915ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned64(a)); 5916ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5917ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5918ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5919ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 8) { 5920ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned64(a)); 5921ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (len >= 8) { 5922ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply64_f__msmcwrite( thr, a ); 5923ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 8; 5924ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 8; 5925ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5926ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned64(a)); 5927ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5928ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5929ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5930ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 4) 5931ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned32(a)); 5932ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 4) { 5933ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply32_f__msmcwrite( thr, a ); 5934ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 4; 5935ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 4; 5936ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5937ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5938ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5939ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 2) 5940ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned16(a)); 5941ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 2) { 5942ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply16_f__msmcwrite( thr, a ); 5943ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 2; 5944ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 2; 5945ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5946ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 5947ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5948ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 1) { 5949ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply08_f__msmcwrite( thr, a ); 5950ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //a += 1; 5951ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 1; 5952ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5953ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(len == 0); 5954ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5955ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5956ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_sapply08_f__msmcread ( Thr* thr, Addr a ) { 5957ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_ac++; 5958ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown STATS__F_SHOW; 5959ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(Filter__ok_to_skip_crd08(thr->filter, a))) { 5960ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_sk++; 5961ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5962ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5963ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply08__msmcread(thr, a); 5964ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5965ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5966ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_sapply16_f__msmcread ( Thr* thr, Addr a ) { 5967ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_ac++; 5968ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown STATS__F_SHOW; 5969ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(Filter__ok_to_skip_crd16(thr->filter, a))) { 5970ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_sk++; 5971ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5972ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5973ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply16__msmcread(thr, a); 5974ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5975ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5976ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_sapply32_f__msmcread ( Thr* thr, Addr a ) { 5977ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_ac++; 5978ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown STATS__F_SHOW; 5979ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(Filter__ok_to_skip_crd32(thr->filter, a))) { 5980ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_sk++; 5981ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5982ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5983ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply32__msmcread(thr, a); 5984ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5985ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5986ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_sapply64_f__msmcread ( Thr* thr, Addr a ) { 5987ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_ac++; 5988ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown STATS__F_SHOW; 5989ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (LIKELY(Filter__ok_to_skip_crd64(thr->filter, a))) { 5990ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__f_sk++; 5991ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 5992ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 5993ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply64__msmcread(thr, a); 5994ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 5995ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 5996ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid zsm_sapplyNN_f__msmcread ( Thr* thr, Addr a, SizeT len ) 5997ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 5998ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* fast track a couple of common cases */ 5999ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 4 && aligned32(a)) { 6000ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply32_f__msmcread( thr, a ); 6001ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 6002ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6003ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 8 && aligned64(a)) { 6004ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply64_f__msmcread( thr, a ); 6005ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 6006ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6007ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6008ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* be completely general (but as efficient as possible) */ 6009ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 6010ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6011ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!aligned16(a) && len >= 1) { 6012ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply08_f__msmcread( thr, a ); 6013ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 1; 6014ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 1; 6015ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned16(a)); 6016ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6017ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 6018ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6019ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!aligned32(a) && len >= 2) { 6020ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply16_f__msmcread( thr, a ); 6021ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 2; 6022ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 2; 6023ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned32(a)); 6024ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6025ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 6026ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6027ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!aligned64(a) && len >= 4) { 6028ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply32_f__msmcread( thr, a ); 6029ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 4; 6030ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 4; 6031ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned64(a)); 6032ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6033ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 6034ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6035ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 8) { 6036ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned64(a)); 6037ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (len >= 8) { 6038ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply64_f__msmcread( thr, a ); 6039ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 8; 6040ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 8; 6041ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6042ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned64(a)); 6043ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6044ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 6045ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6046ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 4) 6047ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned32(a)); 6048ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 4) { 6049ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply32_f__msmcread( thr, a ); 6050ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 4; 6051ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 4; 6052ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6053ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 6054ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6055ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 2) 6056ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(aligned16(a)); 6057ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 2) { 6058ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply16_f__msmcread( thr, a ); 6059ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a += 2; 6060ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 2; 6061ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6062ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len == 0) return; 6063ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6064ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (len >= 1) { 6065ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sapply08_f__msmcread( thr, a ); 6066ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //a += 1; 6067ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len -= 1; 6068ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6069ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(len == 0); 6070ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6071ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6072ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid libhb_Thr_resumes ( Thr* thr ) 6073ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6074ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) VG_(printf)("resume %p\n", thr); 6075ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(thr); 6076b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(!thr->llexit_done); 6077ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Filter__clear(thr->filter, "libhb_Thr_resumes"); 6078ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* A kludge, but .. if this thread doesn't have any marker stacks 6079ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown at all, get one right now. This is easier than figuring out 6080ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown exactly when at thread startup we can and can't take a stack 6081ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown snapshot. */ 6082ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (HG_(clo_history_level) == 1) { 6083ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(thr->local_Kws_n_stacks); 6084ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (VG_(sizeXA)( thr->local_Kws_n_stacks ) == 0) 6085ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown note_local_Kw_n_stack_for(thr); 6086ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6087ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6088ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6089ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6090ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 6091ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 6092ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Synchronisation objects // 6093ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 6094ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 6095ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6096b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* A double linked list of all the SO's. */ 6097b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy StepanovSO* admin_SO = NULL; 6098ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6099b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic SO* SO__Alloc ( void ) 6100b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 6101ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SO* so = HG_(zalloc)( "libhb.SO__Alloc.1", sizeof(SO) ); 6102ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown so->viR = VtsID_INVALID; 6103ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown so->viW = VtsID_INVALID; 6104ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown so->magic = SO_MAGIC; 6105b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Add to double linked list */ 6106b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (admin_SO) { 6107b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(admin_SO->admin_prev == NULL); 6108b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov admin_SO->admin_prev = so; 6109b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov so->admin_next = admin_SO; 6110b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } else { 6111b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov so->admin_next = NULL; 6112b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 6113b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov so->admin_prev = NULL; 6114b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov admin_SO = so; 6115b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* */ 6116ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return so; 6117ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6118b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 6119b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic void SO__Dealloc ( SO* so ) 6120b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 6121ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so); 6122ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so->magic == SO_MAGIC); 6123ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (so->viR == VtsID_INVALID) { 6124ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so->viW == VtsID_INVALID); 6125ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 6126ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so->viW != VtsID_INVALID); 6127ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcdec(so->viR); 6128ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcdec(so->viW); 6129ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6130ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown so->magic = 0; 6131b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Del from double linked list */ 6132b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (so->admin_prev) 6133b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov so->admin_prev->admin_next = so->admin_next; 6134b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (so->admin_next) 6135b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov so->admin_next->admin_prev = so->admin_prev; 6136b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (so == admin_SO) 6137b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov admin_SO = so->admin_next; 6138b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* */ 6139ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free)( so ); 6140ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6141ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6142ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6143ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 6144ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 6145ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Top Level API // 6146ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 6147ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////// 6148ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6149ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void show_thread_state ( HChar* str, Thr* t ) 6150ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6151ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (1) return; 6152ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (t->viR == t->viW) { 6153ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("thr \"%s\" %p has vi* %u==", str, t, t->viR ); 6154ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__pp( t->viR ); 6155ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","\n"); 6156ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 6157ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("thr \"%s\" %p has viR %u==", str, t, t->viR ); 6158ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__pp( t->viR ); 6159ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" viW %u==", t->viW); 6160ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__pp( t->viW ); 6161ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","\n"); 6162ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6163ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6164ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6165ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6166ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownThr* libhb_init ( 6167ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown void (*get_stacktrace)( Thr*, Addr*, UWord ), 6168ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ExeContext* (*get_EC)( Thr* ) 6169ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ) 6170ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6171ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Thr* thr; 6172ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID vi; 6173b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 6174b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // We will have to have to store a large number of these, 6175b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // so make sure they're the size we expect them to be. 6176b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(sizeof(ScalarTS) == 8); 6177b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 6178b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* because first 1024 unusable */ 6179b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(SCALARTS_N_THRBITS >= 11); 6180b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* so as to fit in a UInt w/ 3 bits to spare (see defn of 6181b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Thr_n_RCEC). */ 6182b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(SCALARTS_N_THRBITS <= 29); 6183b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 6184b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Need to be sure that Thr_n_RCEC is 2 words (64-bit) or 3 words 6185b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov (32-bit). It's not correctness-critical, but there are a lot of 6186b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov them, so it's important from a space viewpoint. Unfortunately 6187b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov we simply can't pack it into 2 words on a 32-bit target. */ 6188b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (sizeof(UWord) == 8) { 6189b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(sizeof(Thr_n_RCEC) == 16); 6190b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } else { 6191b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(sizeof(Thr_n_RCEC) == 12); 6192b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 6193b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 6194b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Word sets really are 32 bits. Even on a 64 bit target. */ 6195b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(sizeof(WordSetID) == 4); 6196b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(sizeof(WordSet) == sizeof(WordSetID)); 6197b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 6198ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(get_stacktrace); 6199ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(get_EC); 6200ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown main_get_stacktrace = get_stacktrace; 6201ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown main_get_EC = get_EC; 6202ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6203ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // No need to initialise hg_wordfm. 6204ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // No need to initialise hg_wordset. 6205ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6206b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Allocated once and never deallocated. Used as a temporary in 6207b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS singleton, tick and join operations. */ 6208b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov temp_max_sized_VTS = VTS__new( "libhb.libhb_init.1", ThrID_MAX_VALID ); 6209b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov temp_max_sized_VTS->id = VtsID_INVALID; 6210b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov verydead_thread_table_init(); 6211ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts_set_init(); 6212ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts_tab_init(); 6213ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown event_map_init(); 6214ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__invalidate_caches(); 6215ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6216ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // initialise shadow memory 6217ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_init( SVal__rcinc, SVal__rcdec ); 6218ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6219ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr = Thr__new(); 6220ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vi = VtsID__mk_Singleton( thr, 1 ); 6221ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr->viR = vi; 6222ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr->viW = vi; 6223ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(thr->viR); 6224ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(thr->viW); 6225ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6226ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown show_thread_state(" root", thr); 6227ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return thr; 6228ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6229ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6230ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6231ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownThr* libhb_create ( Thr* parent ) 6232ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6233ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* The child's VTSs are copies of the parent's VTSs, but ticked at 6234ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown the child's index. Since the child's index is guaranteed 6235ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown unique, it has never been seen before, so the implicit value 6236ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown before the tick is zero and after that is one. */ 6237ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Thr* child = Thr__new(); 6238ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6239ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown child->viR = VtsID__tick( parent->viR, child ); 6240ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown child->viW = VtsID__tick( parent->viW, child ); 6241ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Filter__clear(child->filter, "libhb_create(child)"); 6242ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(child->viR); 6243ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(child->viW); 6244ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* We need to do note_local_Kw_n_stack_for( child ), but it's too 6245ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown early for that - it may not have a valid TId yet. So, let 6246ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown libhb_Thr_resumes pick it up the first time the thread runs. */ 6247ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6248ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(VtsID__indexAt( child->viR, child ) == 1); 6249ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(VtsID__indexAt( child->viW, child ) == 1); 6250ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6251ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* and the parent has to move along too */ 6252ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcdec(parent->viR); 6253ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcdec(parent->viW); 6254ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown parent->viR = VtsID__tick( parent->viR, parent ); 6255ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown parent->viW = VtsID__tick( parent->viW, parent ); 6256ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Filter__clear(parent->filter, "libhb_create(parent)"); 6257ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(parent->viR); 6258ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(parent->viW); 6259ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown note_local_Kw_n_stack_for( parent ); 6260ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6261ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown show_thread_state(" child", child); 6262ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown show_thread_state("parent", parent); 6263ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6264ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return child; 6265ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6266ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6267ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Shut down the library, and print stats (in fact that's _all_ 6268ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown this is for. */ 6269ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid libhb_shutdown ( Bool show_stats ) 6270ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6271ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (show_stats) { 6272ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","<<< BEGIN libhb stats >>>\n"); 6273ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" secmaps: %'10lu allocd (%'12lu g-a-range)\n", 6274ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmaps_allocd, 6275ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmap_ga_space_covered); 6276ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" linesZ: %'10lu allocd (%'12lu bytes occupied)\n", 6277ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmap_linesZ_allocd, 6278ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmap_linesZ_bytes); 6279ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" linesF: %'10lu allocd (%'12lu bytes occupied)\n", 6280ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmap_linesF_allocd, 6281ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmap_linesF_bytes); 6282ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" secmaps: %'10lu iterator steppings\n", 6283ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmap_iterator_steppings); 6284ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" secmaps: %'10lu searches (%'12lu slow)\n", 6285ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__secmaps_search, stats__secmaps_search_slow); 6286ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6287ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","\n"); 6288ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" cache: %'lu totrefs (%'lu misses)\n", 6289ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_totrefs, stats__cache_totmisses ); 6290ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" cache: %'14lu Z-fetch, %'14lu F-fetch\n", 6291ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_Z_fetches, stats__cache_F_fetches ); 6292ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" cache: %'14lu Z-wback, %'14lu F-wback\n", 6293ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_Z_wbacks, stats__cache_F_wbacks ); 6294ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" cache: %'14lu invals, %'14lu flushes\n", 6295ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_invals, stats__cache_flushes ); 6296ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" cache: %'14llu arange_New %'14llu direct-to-Zreps\n", 6297ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_make_New_arange, 6298ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cache_make_New_inZrep); 6299ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6300ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","\n"); 6301ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" cline: %'10lu normalises\n", 6302ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_normalises ); 6303ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" cline: c rds 8/4/2/1: %'13lu %'13lu %'13lu %'13lu\n", 6304ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cread64s, 6305ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cread32s, 6306ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cread16s, 6307ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cread08s ); 6308ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" cline: c wrs 8/4/2/1: %'13lu %'13lu %'13lu %'13lu\n", 6309ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cwrite64s, 6310ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cwrite32s, 6311ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cwrite16s, 6312ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_cwrite08s ); 6313ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" cline: s wrs 8/4/2/1: %'13lu %'13lu %'13lu %'13lu\n", 6314ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_swrite64s, 6315ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_swrite32s, 6316ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_swrite16s, 6317ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_swrite08s ); 6318ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" cline: s rd1s %'lu, s copy1s %'lu\n", 6319ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_sread08s, stats__cline_scopy08s ); 6320ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" cline: splits: 8to4 %'12lu 4to2 %'12lu 2to1 %'12lu\n", 6321ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_64to32splits, 6322ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_32to16splits, 6323ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_16to8splits ); 6324ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" cline: pulldowns: 8to4 %'12lu 4to2 %'12lu 2to1 %'12lu\n", 6325ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_64to32pulldown, 6326ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_32to16pulldown, 6327ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cline_16to8pulldown ); 6328ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) 6329ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" cline: sizeof(CacheLineZ) %ld, covers %ld bytes of arange\n", 6330ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Word)sizeof(LineZ), (Word)N_LINE_ARANGE); 6331ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6332ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","\n"); 6333ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6334ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" libhb: %'13llu msmcread (%'llu dragovers)\n", 6335ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__msmcread, stats__msmcread_change); 6336ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" libhb: %'13llu msmcwrite (%'llu dragovers)\n", 6337ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__msmcwrite, stats__msmcwrite_change); 6338ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" libhb: %'13llu cmpLEQ queries (%'llu misses)\n", 6339ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__cmpLEQ_queries, stats__cmpLEQ_misses); 6340ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)(" libhb: %'13llu join2 queries (%'llu misses)\n", 6341ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__join2_queries, stats__join2_misses); 6342ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6343ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","\n"); 6344ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( " libhb: VTSops: tick %'lu, join %'lu, cmpLEQ %'lu\n", 6345ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__vts__tick, stats__vts__join, stats__vts__cmpLEQ ); 6346ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( " libhb: VTSops: cmp_structural %'lu (%'lu slow)\n", 6347ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__vts__cmp_structural, stats__vts__cmp_structural_slow ); 6348b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(printf)( " libhb: VTSset: find__or__clone_and_add %'lu (%'lu allocd)\n", 6349b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov stats__vts_set__focaa, stats__vts_set__focaa_a ); 6350ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( " libhb: VTSops: indexAt_SLOW %'lu\n", 6351ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__vts__indexat_slow ); 6352ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6353ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","\n"); 6354ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( 6355ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown " libhb: %ld entries in vts_table (approximately %lu bytes)\n", 6356ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(sizeXA)( vts_tab ), VG_(sizeXA)( vts_tab ) * sizeof(VtsTE) 6357ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ); 6358ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( " libhb: %lu entries in vts_set\n", 6359ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(sizeFM)( vts_set ) ); 6360ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6361ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","\n"); 6362ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( " libhb: ctxt__rcdec: 1=%lu(%lu eq), 2=%lu, 3=%lu\n", 6363ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_rcdec1, stats__ctxt_rcdec1_eq, 6364ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_rcdec2, 6365ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_rcdec3 ); 6366ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( " libhb: ctxt__rcdec: calls %lu, discards %lu\n", 6367ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_rcdec_calls, stats__ctxt_rcdec_discards); 6368ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( " libhb: contextTab: %lu slots, %lu max ents\n", 6369ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (UWord)N_RCEC_TAB, 6370ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_tab_curr ); 6371ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( " libhb: contextTab: %lu queries, %lu cmps\n", 6372ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_tab_qs, 6373ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stats__ctxt_tab_cmps ); 6374ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#if 0 6375ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(AvlNode) = %lu\n", sizeof(AvlNode)); 6376ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(WordBag) = %lu\n", sizeof(WordBag)); 6377ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(MaybeWord) = %lu\n", sizeof(MaybeWord)); 6378ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(CacheLine) = %lu\n", sizeof(CacheLine)); 6379ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(LineZ) = %lu\n", sizeof(LineZ)); 6380ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(LineF) = %lu\n", sizeof(LineF)); 6381ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(SecMap) = %lu\n", sizeof(SecMap)); 6382ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(Cache) = %lu\n", sizeof(Cache)); 6383ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(SMCacheEnt) = %lu\n", sizeof(SMCacheEnt)); 6384ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(CountedSVal) = %lu\n", sizeof(CountedSVal)); 6385ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(VTS) = %lu\n", sizeof(VTS)); 6386ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(ScalarTS) = %lu\n", sizeof(ScalarTS)); 6387ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(VtsTE) = %lu\n", sizeof(VtsTE)); 6388ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(MSMInfo) = %lu\n", sizeof(MSMInfo)); 6389ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6390ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(struct _XArray) = %lu\n", sizeof(struct _XArray)); 6391ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(struct _WordFM) = %lu\n", sizeof(struct _WordFM)); 6392ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(struct _Thr) = %lu\n", sizeof(struct _Thr)); 6393ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sizeof(struct _SO) = %lu\n", sizeof(struct _SO)); 6394ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#endif 6395ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6396ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","<<< END libhb stats >>>\n"); 6397ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","\n"); 6398ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6399ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6400ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6401ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6402b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Receive notification that a thread has low level exited. The 6403b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov significance here is that we do not expect to see any more memory 6404b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov references from it. */ 6405ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid libhb_async_exit ( Thr* thr ) 6406ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6407ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(thr); 6408b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(!thr->llexit_done); 6409b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thr->llexit_done = True; 6410ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6411ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* free up Filter and local_Kws_n_stacks (well, actually not the 6412ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown latter ..) */ 6413ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(thr->filter); 6414ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HG_(free)(thr->filter); 6415ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr->filter = NULL; 6416ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6417b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Tell the VTS mechanism this thread has exited, so it can 6418b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov participate in VTS pruning. Note this can only happen if the 6419b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thread has both ll_exited and has been joined with. */ 6420b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (thr->joinedwith_done) 6421b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS__declare_thread_very_dead(thr); 6422b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 6423ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Another space-accuracy tradeoff. Do we want to be able to show 6424ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown H1 history for conflicts in threads which have since exited? If 6425ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown yes, then we better not free up thr->local_Kws_n_stacks. The 6426ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown downside is a potential per-thread leak of up to 6427ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown N_KWs_N_STACKs_PER_THREAD * sizeof(ULong_n_EC) * whatever the 6428ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown XArray average overcommit factor is (1.5 I'd guess). */ 6429ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // hence: 6430ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // VG_(deleteXA)(thr->local_Kws_n_stacks); 6431ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // thr->local_Kws_n_stacks = NULL; 6432ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6433ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6434b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov/* Receive notification that a thread has been joined with. The 6435b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov significance here is that we do not expect to see any further 6436b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov references to its vector clocks (Thr::viR and Thr::viW). */ 6437b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovvoid libhb_joinedwith_done ( Thr* thr ) 6438b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 6439b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(thr); 6440b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Caller must ensure that this is only ever called once per Thr. */ 6441b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(!thr->joinedwith_done); 6442b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thr->joinedwith_done = True; 6443b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (thr->llexit_done) 6444b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VTS__declare_thread_very_dead(thr); 6445b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 6446b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 6447b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 6448ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Both Segs and SOs point to VTSs. However, there is no sharing, so 6449ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a Seg that points at a VTS is its one-and-only owner, and ditto for 6450ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a SO that points at a VTS. */ 6451ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6452ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownSO* libhb_so_alloc ( void ) 6453ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6454ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return SO__Alloc(); 6455ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6456ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6457ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid libhb_so_dealloc ( SO* so ) 6458ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6459ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so); 6460ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so->magic == SO_MAGIC); 6461ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SO__Dealloc(so); 6462ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6463ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6464ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* See comments in libhb.h for details on the meaning of 6465ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown strong vs weak sends and strong vs weak receives. */ 6466ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid libhb_so_send ( Thr* thr, SO* so, Bool strong_send ) 6467ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6468ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Copy the VTSs from 'thr' into the sync object, and then move 6469ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown the thread along one step. */ 6470ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6471ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so); 6472ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so->magic == SO_MAGIC); 6473ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6474ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* stay sane .. a thread's read-clock must always lead or be the 6475ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown same as its write-clock */ 6476ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown { Bool leq = VtsID__cmpLEQ(thr->viW, thr->viR); 6477ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(leq); 6478ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6479ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6480ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* since we're overwriting the VtsIDs in the SO, we need to drop 6481ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown any references made by the previous contents thereof */ 6482ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (so->viR == VtsID_INVALID) { 6483ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so->viW == VtsID_INVALID); 6484ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown so->viR = thr->viR; 6485ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown so->viW = thr->viW; 6486ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(so->viR); 6487ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(so->viW); 6488ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 6489ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* In a strong send, we dump any previous VC in the SO and 6490ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown install the sending thread's VC instead. For a weak send we 6491ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown must join2 with what's already there. */ 6492ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so->viW != VtsID_INVALID); 6493ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcdec(so->viR); 6494ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcdec(so->viW); 6495ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown so->viR = strong_send ? thr->viR : VtsID__join2( so->viR, thr->viR ); 6496ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown so->viW = strong_send ? thr->viW : VtsID__join2( so->viW, thr->viW ); 6497ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(so->viR); 6498ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(so->viW); 6499ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6500ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6501ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* move both parent clocks along */ 6502ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcdec(thr->viR); 6503ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcdec(thr->viW); 6504ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr->viR = VtsID__tick( thr->viR, thr ); 6505ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr->viW = VtsID__tick( thr->viW, thr ); 6506b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (!thr->llexit_done) { 6507ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Filter__clear(thr->filter, "libhb_so_send"); 6508ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown note_local_Kw_n_stack_for(thr); 6509ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6510ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(thr->viR); 6511ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(thr->viW); 6512ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6513ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (strong_send) 6514ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown show_thread_state("s-send", thr); 6515ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown else 6516ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown show_thread_state("w-send", thr); 6517ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6518ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6519ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid libhb_so_recv ( Thr* thr, SO* so, Bool strong_recv ) 6520ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6521ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so); 6522ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so->magic == SO_MAGIC); 6523ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6524ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (so->viR != VtsID_INVALID) { 6525ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so->viW != VtsID_INVALID); 6526ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6527ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Weak receive (basically, an R-acquisition of a R-W lock). 6528ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown This advances the read-clock of the receiver, but not the 6529ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown write-clock. */ 6530ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcdec(thr->viR); 6531ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr->viR = VtsID__join2( thr->viR, so->viR ); 6532ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(thr->viR); 6533ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6534ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* At one point (r10589) it seemed safest to tick the clocks for 6535ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown the receiving thread after the join. But on reflection, I 6536ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown wonder if that might cause it to 'overtake' constraints, 6537ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown which could lead to missing races. So, back out that part of 6538ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown r10589. */ 6539ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //VtsID__rcdec(thr->viR); 6540ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //thr->viR = VtsID__tick( thr->viR, thr ); 6541ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //VtsID__rcinc(thr->viR); 6542ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6543ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* For a strong receive, we also advance the receiver's write 6544ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown clock, which means the receive as a whole is essentially 6545ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown equivalent to a W-acquisition of a R-W lock. */ 6546ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (strong_recv) { 6547ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcdec(thr->viW); 6548ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thr->viW = VtsID__join2( thr->viW, so->viW ); 6549ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VtsID__rcinc(thr->viW); 6550ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6551ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* See comment just above, re r10589. */ 6552ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //VtsID__rcdec(thr->viW); 6553ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //thr->viW = VtsID__tick( thr->viW, thr ); 6554ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //VtsID__rcinc(thr->viW); 6555ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6556ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6557ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (thr->filter) 6558ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Filter__clear(thr->filter, "libhb_so_recv"); 6559ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown note_local_Kw_n_stack_for(thr); 6560ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6561ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (strong_recv) 6562ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown show_thread_state("s-recv", thr); 6563ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown else 6564ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown show_thread_state("w-recv", thr); 6565ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6566ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 6567ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so->viW == VtsID_INVALID); 6568ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Deal with degenerate case: 'so' has no vts, so there has been 6569ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown no message posted to it. Just ignore this case. */ 6570ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown show_thread_state("d-recv", thr); 6571ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6572ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6573ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6574ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownBool libhb_so_everSent ( SO* so ) 6575ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6576ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (so->viR == VtsID_INVALID) { 6577ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so->viW == VtsID_INVALID); 6578ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 6579ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 6580ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(so->viW != VtsID_INVALID); 6581ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return True; 6582ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 6583ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6584ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6585ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define XXX1 0 // 0x67a106c 6586ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define XXX2 0 6587ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6588ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic inline Bool TRACEME(Addr a, SizeT szB) { 6589ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (XXX1 && a <= XXX1 && XXX1 <= a+szB) return True; 6590ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (XXX2 && a <= XXX2 && XXX2 <= a+szB) return True; 6591ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return False; 6592ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6593ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void trace ( Thr* thr, Addr a, SizeT szB, HChar* s ) { 6594ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal sv = zsm_sread08(a); 6595ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("thr %p (%#lx,%lu) %s: 0x%016llx ", thr,a,szB,s,sv); 6596ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown show_thread_state("", thr); 6597ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%s","\n"); 6598ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6599ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6600ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid libhb_srange_new ( Thr* thr, Addr a, SizeT szB ) 6601ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6602ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal sv = SVal__mkC(thr->viW, thr->viW); 6603ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_SVal_C(sv)); 6604ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0 && TRACEME(a,szB)) trace(thr,a,szB,"nw-before"); 6605ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sset_range( a, szB, sv ); 6606ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Filter__clear_range( thr->filter, a, szB ); 6607ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0 && TRACEME(a,szB)) trace(thr,a,szB,"nw-after "); 6608ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6609ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6610b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovvoid libhb_srange_noaccess_NoFX ( Thr* thr, Addr a, SizeT szB ) 6611ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6612ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* do nothing */ 6613ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6614ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6615b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovvoid libhb_srange_noaccess_AHAE ( Thr* thr, Addr a, SizeT szB ) 6616b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 6617b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* This really does put the requested range in NoAccess. It's 6618b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov expensive though. */ 6619b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov SVal sv = SVal_NOACCESS; 6620b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov tl_assert(is_sane_SVal_C(sv)); 6621b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov zsm_sset_range( a, szB, sv ); 6622b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Filter__clear_range( thr->filter, a, szB ); 6623b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 6624b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 6625ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid libhb_srange_untrack ( Thr* thr, Addr a, SizeT szB ) 6626ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6627ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SVal sv = SVal_NOACCESS; 6628ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(is_sane_SVal_C(sv)); 6629ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0 && TRACEME(a,szB)) trace(thr,a,szB,"untrack-before"); 6630ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_sset_range( a, szB, sv ); 6631ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Filter__clear_range( thr->filter, a, szB ); 6632ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0 && TRACEME(a,szB)) trace(thr,a,szB,"untrack-after "); 6633ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6634ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6635b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy StepanovThread* libhb_get_Thr_hgthread ( Thr* thr ) { 6636ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(thr); 6637b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return thr->hgthread; 6638ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6639ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6640b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovvoid libhb_set_Thr_hgthread ( Thr* thr, Thread* hgthread ) { 6641ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(thr); 6642b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov thr->hgthread = hgthread; 6643ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6644ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6645ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid libhb_copy_shadow_state ( Thr* thr, Addr src, Addr dst, SizeT len ) 6646ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6647ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zsm_scopy_range(src, dst, len); 6648ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Filter__clear_range( thr->filter, dst, len ); 6649ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6650ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6651ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid libhb_maybe_GC ( void ) 6652ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 6653ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown event_map_maybe_GC(); 6654ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* If there are still freelist entries available, no need for a 6655ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown GC. */ 6656ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (vts_tab_freelist != VtsID_INVALID) 6657ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 6658ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* So all the table entries are full, and we're having to expand 6659ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown the table. But did we hit the threshhold point yet? */ 6660ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (VG_(sizeXA)( vts_tab ) < vts_next_GC_at) 6661ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 6662ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vts_tab__do_GC( False/*don't show stats*/ ); 6663ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 6664ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6665ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6666ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 6667ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 6668ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 6669ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// SECTION END main library // 6670ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// // 6671ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 6672ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown///////////////////////////////////////////////////////////////// 6673ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6674ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--------------------------------------------------------------------*/ 6675ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--- end libhb_main.c ---*/ 6676ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--------------------------------------------------------------------*/ 6677