m_transtab.c revision 0b9d74abd0a663b530d290b2b788ddeda46e5400
1de4a1d01951937632098a6cda45859afa587a06fsewardj 2de4a1d01951937632098a6cda45859afa587a06fsewardj/*--------------------------------------------------------------------*/ 3de4a1d01951937632098a6cda45859afa587a06fsewardj/*--- Management of the translation table and cache. ---*/ 48bddf58af8cc7342d4bde6712c5a6a33bf2850d4njn/*--- m_transtab.c ---*/ 5de4a1d01951937632098a6cda45859afa587a06fsewardj/*--------------------------------------------------------------------*/ 6de4a1d01951937632098a6cda45859afa587a06fsewardj 7de4a1d01951937632098a6cda45859afa587a06fsewardj/* 8b9c427c63a278cc612ae0ec573be7bb1abaa447fnjn This file is part of Valgrind, a dynamic binary instrumentation 9b9c427c63a278cc612ae0ec573be7bb1abaa447fnjn framework. 10de4a1d01951937632098a6cda45859afa587a06fsewardj 11e4b0bf07b0ee0a18eacc5aba91686ab5fc1d327bsewardj Copyright (C) 2000-2006 Julian Seward 12de4a1d01951937632098a6cda45859afa587a06fsewardj jseward@acm.org 13de4a1d01951937632098a6cda45859afa587a06fsewardj 14de4a1d01951937632098a6cda45859afa587a06fsewardj This program is free software; you can redistribute it and/or 15de4a1d01951937632098a6cda45859afa587a06fsewardj modify it under the terms of the GNU General Public License as 16de4a1d01951937632098a6cda45859afa587a06fsewardj published by the Free Software Foundation; either version 2 of the 17de4a1d01951937632098a6cda45859afa587a06fsewardj License, or (at your option) any later version. 18de4a1d01951937632098a6cda45859afa587a06fsewardj 19de4a1d01951937632098a6cda45859afa587a06fsewardj This program is distributed in the hope that it will be useful, but 20de4a1d01951937632098a6cda45859afa587a06fsewardj WITHOUT ANY WARRANTY; without even the implied warranty of 21de4a1d01951937632098a6cda45859afa587a06fsewardj MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 22de4a1d01951937632098a6cda45859afa587a06fsewardj General Public License for more details. 23de4a1d01951937632098a6cda45859afa587a06fsewardj 24de4a1d01951937632098a6cda45859afa587a06fsewardj You should have received a copy of the GNU General Public License 25de4a1d01951937632098a6cda45859afa587a06fsewardj along with this program; if not, write to the Free Software 26de4a1d01951937632098a6cda45859afa587a06fsewardj Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 27de4a1d01951937632098a6cda45859afa587a06fsewardj 02111-1307, USA. 28de4a1d01951937632098a6cda45859afa587a06fsewardj 29e49d8e7dfd3a9c96feb9935b5920973dfc0b170anjn The GNU General Public License is contained in the file COPYING. 30de4a1d01951937632098a6cda45859afa587a06fsewardj*/ 31de4a1d01951937632098a6cda45859afa587a06fsewardj 32c7561b931e249acf3768ead77638545b0ccaa8f1njn#include "pub_core_basics.h" 3345f4e7c91119c7d01a59f5e827c67841632c9314sewardj#include "pub_core_debuglog.h" 341f0d814045aba94e01e62e04e968ca8b970b3d44cerion#include "pub_core_machine.h" // For VG(machine_get_VexArchInfo) 3597405b2d134b52880d6dbec3eb2929e2002c2542njn#include "pub_core_libcbase.h" 36132bfccd21960e462352175f8553a5bdce8a210cnjn#include "pub_core_libcassert.h" 3736a20fa5f779a0a6fb7b4a90dcaa6376481f1faanjn#include "pub_core_libcprint.h" 382024234c590f408994b373abfb00bc2cd2a90c48njn#include "pub_core_options.h" 3910f08cf5b84882eebbb6712a7be890577650e8adsewardj#include "pub_core_tooliface.h" // For VG_(details).avg_translation_sizeB 408bddf58af8cc7342d4bde6712c5a6a33bf2850d4njn#include "pub_core_transtab.h" 4145f4e7c91119c7d01a59f5e827c67841632c9314sewardj#include "pub_core_aspacemgr.h" 4245f4e7c91119c7d01a59f5e827c67841632c9314sewardj#include "pub_core_mallocfree.h" // VG_(out_of_memory_NORETURN) 43de4a1d01951937632098a6cda45859afa587a06fsewardj 4418d7513cc08bf982711c8a22b70d56af6aa87b33sewardj/* #define DEBUG_TRANSTAB */ 4518d7513cc08bf982711c8a22b70d56af6aa87b33sewardj 46de4a1d01951937632098a6cda45859afa587a06fsewardj 476c3769f487145a08c01b58d6e5db3ba274062ad4sewardj/*-------------------------------------------------------------*/ 486c3769f487145a08c01b58d6e5db3ba274062ad4sewardj/*--- Management of the FIFO-based translation table+cache. ---*/ 496c3769f487145a08c01b58d6e5db3ba274062ad4sewardj/*-------------------------------------------------------------*/ 50de4a1d01951937632098a6cda45859afa587a06fsewardj 516c3769f487145a08c01b58d6e5db3ba274062ad4sewardj/*------------------ CONSTANTS ------------------*/ 52de4a1d01951937632098a6cda45859afa587a06fsewardj 53fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/* Number of sectors the TC is divided into. If you need a larger 54fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj overall translation cache, increase this value. */ 55fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj#define N_SECTORS 8 56de4a1d01951937632098a6cda45859afa587a06fsewardj 57fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/* Number of TC entries in each sector. This needs to be a prime 586c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj number to work properly, it must be <= 65535 (so that a TT index 596c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj fits in a UShort, leaving room for 0xFFFF(EC2TTE_DELETED) to denote 606c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 'deleted') and it is strongly recommended not to change this. 616c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 65521 is the largest prime <= 65535. */ 626c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj#define N_TTES_PER_SECTOR /*30011*/ /*40009*/ 65521 63de4a1d01951937632098a6cda45859afa587a06fsewardj 64fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/* Because each sector contains a hash table of TTEntries, we need to 65fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj specify the maximum allowable loading, after which the sector is 66fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj deemed full. */ 676c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj#define SECTOR_TT_LIMIT_PERCENT 80 68de4a1d01951937632098a6cda45859afa587a06fsewardj 69fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/* The sector is deemed full when this many entries are in it. */ 70fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj#define N_TTES_PER_SECTOR_USABLE \ 71fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj ((N_TTES_PER_SECTOR * SECTOR_TT_LIMIT_PERCENT) / 100) 726c3769f487145a08c01b58d6e5db3ba274062ad4sewardj 736c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj/* Equivalence classes for fast address range deletion. There are 1 + 746c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 2^ECLASS_WIDTH bins. The highest one, ECLASS_MISC, describes an 756c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj address range which does not fall cleanly within any specific bin. 766c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Note that ECLASS_SHIFT + ECLASS_WIDTH must be < 32. */ 776c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj#define ECLASS_SHIFT 11 786c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj#define ECLASS_WIDTH 8 796c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj#define ECLASS_MISC (1 << ECLASS_WIDTH) 806c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj#define ECLASS_N (1 + ECLASS_MISC) 816c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 826c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj#define EC2TTE_DELETED 0xFFFF /* 16-bit special value */ 836c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 84de4a1d01951937632098a6cda45859afa587a06fsewardj 856c3769f487145a08c01b58d6e5db3ba274062ad4sewardj/*------------------ TYPES ------------------*/ 86de4a1d01951937632098a6cda45859afa587a06fsewardj 87fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/* A translation-cache entry is two parts: 88fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj - The guest address of the first (entry) bb in the translation, 89fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj as a 64-bit word. 90fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj - One or more 64-bit words containing the code. 91fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj It is supposed to be 64-bit aligned. 92fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj*/ 93fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/* 946c3769f487145a08c01b58d6e5db3ba274062ad4sewardjtypedef 95fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj struct { 96fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj Addr64 orig_addr; 97fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj ULong code[0]; 986c3769f487145a08c01b58d6e5db3ba274062ad4sewardj } 996c3769f487145a08c01b58d6e5db3ba274062ad4sewardj TCEntry; 100fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj*/ 1016c3769f487145a08c01b58d6e5db3ba274062ad4sewardj 102fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/* A translation-table entry. This indicates precisely which areas of 103fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj guest code are included in the translation, and contains all other 104fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj auxiliary info too. */ 1056c3769f487145a08c01b58d6e5db3ba274062ad4sewardjtypedef 1066c3769f487145a08c01b58d6e5db3ba274062ad4sewardj struct { 107fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* Profiling only: the count and weight (arbitrary meaning) for 108fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj this translation. Weight is a property of the translation 109fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj itself and computed once when the translation is created. 110fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj Count is an entry count for the translation and is 111fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj incremented by 1 every time the translation is used, if we 112fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj are profiling. */ 113fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj UInt count; 114fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj UShort weight; 115fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 116fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* Status of the slot. Note, we need to be able to do lazy 117fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj deletion, hence the Deleted state. */ 118fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj enum { InUse, Deleted, Empty } status; 119fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 120fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* Pointer to the corresponding TCEntry (must be in the same 121fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj sector!) */ 122fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj ULong* tce; 123fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 124fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* This is the original guest address that purportedly is the 125fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj entry point of the translation. You might think that .entry 126fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj should be the same as .vge->base[0], and most of the time it 127fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj is. However, when doing redirections, that is not the case. 128fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj .vge must always correctly describe the guest code sections 129fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj from which this translation was made. However, .entry may or 130fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj may not be a lie, depending on whether or not we're doing 131fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj redirection. */ 132fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj Addr64 entry; 133fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 134fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* This structure describes precisely what ranges of guest code 135fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj the translation covers, so we can decide whether or not to 136fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj delete it when translations of a given address range are 137fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj invalidated. */ 138fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj VexGuestExtents vge; 1396c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 1406c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* Address range summary info: these are pointers back to 1416c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj eclass[] entries in the containing Sector. Those entries in 1426c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj turn point back here -- the two structures are mutually 1436c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj redundant but both necessary to make fast deletions work. 1446c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj The eclass info is similar to, and derived from, this entry's 1456c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 'vge' field, but it is not the same */ 1466c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj UShort n_tte2ec; // # tte2ec pointers (1 to 3) 1476c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj UShort tte2ec_ec[3]; // for each, the eclass # 1486c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj UInt tte2ec_ix[3]; // and the index within the eclass. 1496c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj // for i in 0 .. n_tte2ec-1 1506c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj // sec->ec2tte[ tte2ec_ec[i] ][ tte2ec_ix[i] ] 1516c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj // should be the index 1526c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj // of this TTEntry in the containing Sector's tt array. 1536c3769f487145a08c01b58d6e5db3ba274062ad4sewardj } 1546c3769f487145a08c01b58d6e5db3ba274062ad4sewardj TTEntry; 1556c3769f487145a08c01b58d6e5db3ba274062ad4sewardj 1564ccf707a1b52898b878fc90dc0054a0c23cc9fccsewardj 157fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/* Finally, a sector itself. Each sector contains an array of 158fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj TCEntries, which hold code, and an array of TTEntries, containing 159fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj all required administrative info. Profiling is supported using the 160fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj TTEntry .count and .weight fields, if required. Each sector is 161fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj independent in that no cross-sector references are allowed. 1624ccf707a1b52898b878fc90dc0054a0c23cc9fccsewardj 163fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj If the sector is not in use, all three pointers are NULL and 164fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj tt_n_inuse is zero. 165fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj*/ 166fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjtypedef 167fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj struct { 168fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* The TCEntry area. Size of this depends on the average 169fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj translation size. We try and size it so it becomes full 170fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj precisely when this sector's translation table (tt) reaches 171fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj its load limit (SECTOR_TT_LIMIT_PERCENT). */ 172fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj ULong* tc; 1734ccf707a1b52898b878fc90dc0054a0c23cc9fccsewardj 174fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* The TTEntry array. This is a fixed size, always containing 175fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj exactly N_TTES_PER_SECTOR entries. */ 176fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj TTEntry* tt; 1776c3769f487145a08c01b58d6e5db3ba274062ad4sewardj 178fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* This points to the current allocation point in tc. */ 179fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj ULong* tc_next; 180de4a1d01951937632098a6cda45859afa587a06fsewardj 181fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* The count of tt entries with state InUse. */ 182fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj Int tt_n_inuse; 1836c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 1846c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* Expandable arrays of tt indices for each of the ECLASS_N 1856c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj address range equivalence classes. These hold indices into 1866c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj the containing sector's tt array, which in turn should point 1876c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj back here. */ 1886c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Int ec2tte_size[ECLASS_N]; 1896c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Int ec2tte_used[ECLASS_N]; 1906c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj UShort* ec2tte[ECLASS_N]; 191fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj } 192fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj Sector; 193de4a1d01951937632098a6cda45859afa587a06fsewardj 194de4a1d01951937632098a6cda45859afa587a06fsewardj 1956c3769f487145a08c01b58d6e5db3ba274062ad4sewardj/*------------------ DECLS ------------------*/ 1966c3769f487145a08c01b58d6e5db3ba274062ad4sewardj 197fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/* The root data structure is an array of sectors. The index of the 198fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj youngest sector is recorded, and new translations are put into that 199fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj sector. When it fills up, we move along to the next sector and 200fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj start to fill that up, wrapping around at the end of the array. 201fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj That way, once all N_TC_SECTORS have been bought into use for the 202fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj first time, and are full, we then re-use the oldest sector, 203fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj endlessly. 204fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 205fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj When running, youngest sector should be between >= 0 and < 206fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj N_TC_SECTORS. The initial -1 value indicates the TT/TC system is 207fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj not yet initialised. 208fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj*/ 209fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjstatic Sector sectors[N_SECTORS]; 210fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjstatic Int youngest_sector = -1; 2116c3769f487145a08c01b58d6e5db3ba274062ad4sewardj 212fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/* The number of ULongs in each TCEntry area. This is computed once 213fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj at startup and does not change. */ 214fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjstatic Int tc_sector_szQ; 2156c3769f487145a08c01b58d6e5db3ba274062ad4sewardj 2166c3769f487145a08c01b58d6e5db3ba274062ad4sewardj 217fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/* Fast helper for the TC. A direct-mapped cache which holds a 218fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj pointer to a TC entry which may or may not be the correct one, but 219fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj which we hope usually is. This array is referred to directly from 220fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj <arch>/dispatch.S. 2216c3769f487145a08c01b58d6e5db3ba274062ad4sewardj 222fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj Entries in tt_fast may point to any valid TC entry, regardless of 223fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj which sector it's in. Consequently we must be very careful to 224fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj invalidate this cache when TC entries are changed or disappear. 225c0d8f68810eb79a5a6cb6049d6a4ab1de7dc4e18sewardj 226fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj A special TCEntry -- bogus_tc_entry -- must be pointed at to cause 227fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj that cache entry to miss. This relies on the assumption that no 228fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj guest code actually has an address of 0x1. 229fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj*/ 230fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/*global*/ ULong* VG_(tt_fast)[VG_TT_FAST_SIZE]; 23192e7b7f3d93564e293483fcf0f52f2d6f6c6dcc0nethercote 232fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjstatic ULong bogus_tc_entry = (Addr64)1; 23392e7b7f3d93564e293483fcf0f52f2d6f6c6dcc0nethercote 23492e7b7f3d93564e293483fcf0f52f2d6f6c6dcc0nethercote 235fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/* For profiling, we have a parallel array of pointers to .count 236fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj fields in TT entries. Again, these pointers must be invalidated 237fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj when translations disappear. A NULL pointer suffices to indicate 238fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj an unused slot. 2396c3769f487145a08c01b58d6e5db3ba274062ad4sewardj 240fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj tt_fast and tt_fastN change together: if tt_fast[i] points to 241fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj bogus_tc_entry then the corresponding tt_fastN[i] must be null. If 242fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj tt_fast[i] points to some TC entry somewhere, then tt_fastN[i] 243fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj *must* point to the .count field of the corresponding TT entry. 2446c3769f487145a08c01b58d6e5db3ba274062ad4sewardj 245fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj tt_fast and tt_fastN are referred to from assembly code 246fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj (dispatch.S). 247fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj*/ 248fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/*global*/ UInt* VG_(tt_fastN)[VG_TT_FAST_SIZE]; 249de4a1d01951937632098a6cda45859afa587a06fsewardj 250de4a1d01951937632098a6cda45859afa587a06fsewardj 251663a1bd7a8def9d4ef4a10615732a15cc9b09171sewardj/* Make sure we're not used before initialisation. */ 252663a1bd7a8def9d4ef4a10615732a15cc9b09171sewardjstatic Bool init_done = False; 253663a1bd7a8def9d4ef4a10615732a15cc9b09171sewardj 254663a1bd7a8def9d4ef4a10615732a15cc9b09171sewardj 255fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/*------------------ STATS DECLS ------------------*/ 256de4a1d01951937632098a6cda45859afa587a06fsewardj 257fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/* Number of fast-cache updates and flushes done. */ 258fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjULong n_fast_flushes = 0; 259fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjULong n_fast_updates = 0; 26022854b907c1925a9af97ad20ca3dacb5e300d6f9sewardj 261fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/* Number of full lookups done. */ 262fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjULong n_full_lookups = 0; 263fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjULong n_lookup_probes = 0; 26422854b907c1925a9af97ad20ca3dacb5e300d6f9sewardj 26526412bd2c4ef3e55683141f132ebd1eb32d8530bsewardj/* Number/osize/tsize of translations entered; also the number of 26626412bd2c4ef3e55683141f132ebd1eb32d8530bsewardj those for which self-checking was requested. */ 26726412bd2c4ef3e55683141f132ebd1eb32d8530bsewardjULong n_in_count = 0; 26826412bd2c4ef3e55683141f132ebd1eb32d8530bsewardjULong n_in_osize = 0; 26926412bd2c4ef3e55683141f132ebd1eb32d8530bsewardjULong n_in_tsize = 0; 27026412bd2c4ef3e55683141f132ebd1eb32d8530bsewardjULong n_in_sc_count = 0; 271de4a1d01951937632098a6cda45859afa587a06fsewardj 272fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/* Number/osize of translations discarded due to lack of space. */ 273fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjULong n_dump_count = 0; 274fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjULong n_dump_osize = 0; 275fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 276fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/* Number/osize of translations discarded due to requests to do so. */ 277fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjULong n_disc_count = 0; 278fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjULong n_disc_osize = 0; 279fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 280fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 2816c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj/*-------------------------------------------------------------*/ 2826c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj/*--- Address-range equivalence class stuff ---*/ 2836c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj/*-------------------------------------------------------------*/ 2846c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 2856c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj/* Return equivalence class number for a range. */ 2866c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 2876c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardjstatic Int range_to_eclass ( Addr64 start, UInt len ) 2886c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj{ 2896c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj UInt mask = (1 << ECLASS_WIDTH) - 1; 2906c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj UInt lo = (UInt)start; 2916c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj UInt hi = lo + len - 1; 2926c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj UInt loBits = (lo >> ECLASS_SHIFT) & mask; 2936c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj UInt hiBits = (hi >> ECLASS_SHIFT) & mask; 2946c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (loBits == hiBits) { 2956c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(loBits < ECLASS_N-1); 2966c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj return loBits; 2976c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } else { 2986c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj return ECLASS_MISC; 2996c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 3006c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj} 3016c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 3026c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 3036c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj/* Calculates the equivalence class numbers for any VexGuestExtent. 3046c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj These are written in *eclasses, which must be big enough to hold 3 3056c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Ints. The number written, between 1 and 3, is returned. The 3066c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj eclasses are presented in order, and any duplicates are removed. 3076c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj*/ 3086c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 3096c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardjstatic 3106c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardjInt vexGuestExtents_to_eclasses ( /*OUT*/Int* eclasses, 3116c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj VexGuestExtents* vge ) 3126c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj{ 3136c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj# define SWAP(_lv1,_lv2) \ 3146c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj do { Int t = _lv1; _lv1 = _lv2; _lv2 = t; } while (0) 3156c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 3166c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Int i, j, n_ec, r; 3176c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 3186c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(vge->n_used >= 1 && vge->n_used <= 3); 3196c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 3206c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj n_ec = 0; 3216c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj for (i = 0; i < vge->n_used; i++) { 3226c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj r = range_to_eclass( vge->base[i], (UInt)vge->len[i] ); 3236c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (r == ECLASS_MISC) 3246c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj goto bad; 3256c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* only add if we haven't already seen it */ 3266c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj for (j = 0; j < n_ec; j++) 3276c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (eclasses[j] == r) 3286c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj break; 3296c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (j == n_ec) 3306c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj eclasses[n_ec++] = r; 3316c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 3326c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 3336c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (n_ec == 1) 3346c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj return 1; 3356c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 3366c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (n_ec == 2) { 3376c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* sort */ 3386c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (eclasses[0] > eclasses[1]) 3396c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj SWAP(eclasses[0], eclasses[1]); 3406c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj return 2; 3416c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 3426c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 3436c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (n_ec == 3) { 3446c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* sort */ 3456c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (eclasses[0] > eclasses[2]) 3466c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj SWAP(eclasses[0], eclasses[2]); 3476c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (eclasses[0] > eclasses[1]) 3486c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj SWAP(eclasses[0], eclasses[1]); 3496c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (eclasses[1] > eclasses[2]) 3506c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj SWAP(eclasses[1], eclasses[2]); 3516c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj return 3; 3526c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 3536c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 3546c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* NOTREACHED */ 3556c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(0); 3566c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 3576c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj bad: 3586c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj eclasses[0] = ECLASS_MISC; 3596c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj return 1; 3606c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 3616c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj# undef SWAP 3626c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj} 3636c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 3646c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 3656c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj/* Add tteno to the set of entries listed for equivalence class ec in 3666c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj this sector. Returns used location in eclass array. */ 3676c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 3686c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardjstatic 3696c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardjUInt addEClassNo ( /*MOD*/Sector* sec, Int ec, UShort tteno ) 3706c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj{ 3716c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Int old_sz, new_sz, i, r; 3726c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj UShort *old_ar, *new_ar; 3736c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 3746c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(ec >= 0 && ec < ECLASS_N); 3756c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(tteno < N_TTES_PER_SECTOR); 3766c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 3776c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (0) VG_(printf)("ec %d gets %d\n", ec, (Int)tteno); 3786c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 3796c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (sec->ec2tte_used[ec] >= sec->ec2tte_size[ec]) { 3806c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 3816c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(sec->ec2tte_used[ec] == sec->ec2tte_size[ec]); 3826c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 3836c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj old_sz = sec->ec2tte_size[ec]; 3846c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj old_ar = sec->ec2tte[ec]; 3856c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj new_sz = old_sz==0 ? 8 : old_sz<64 ? 2*old_sz : (3*old_sz)/2; 3866c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj new_ar = VG_(arena_malloc)(VG_AR_TTAUX, new_sz * sizeof(UShort)); 3876c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj for (i = 0; i < old_sz; i++) 3886c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj new_ar[i] = old_ar[i]; 3896c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (old_ar) 3906c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj VG_(arena_free)(VG_AR_TTAUX, old_ar); 3916c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sec->ec2tte_size[ec] = new_sz; 3926c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sec->ec2tte[ec] = new_ar; 3936c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 3946c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (0) VG_(printf)("expand ec %d to %d\n", ec, new_sz); 3956c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 3966c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 3976c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* Common case */ 3986c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj r = sec->ec2tte_used[ec]++; 3996c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(r >= 0 && r < sec->ec2tte_size[ec]); 4006c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sec->ec2tte[ec][r] = tteno; 4016c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj return (UInt)r; 4026c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj} 4036c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 4046c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 4056c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj/* 'vge' is being added to 'sec' at TT entry 'tteno'. Add appropriate 4066c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj eclass entries to 'sec'. */ 4076c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 4086c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardjstatic 4096c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardjvoid upd_eclasses_after_add ( /*MOD*/Sector* sec, Int tteno ) 4106c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj{ 4116c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Int i, r, eclasses[3]; 4126c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj TTEntry* tte; 4136c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(tteno >= 0 && tteno < N_TTES_PER_SECTOR); 4146c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 4156c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj tte = &sec->tt[tteno]; 4166c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj r = vexGuestExtents_to_eclasses( eclasses, &tte->vge ); 4176c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 4186c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(r >= 1 && r <= 3); 4196c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj tte->n_tte2ec = r; 4206c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 4216c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj for (i = 0; i < r; i++) { 4226c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj tte->tte2ec_ec[i] = eclasses[i]; 4236c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj tte->tte2ec_ix[i] = addEClassNo( sec, eclasses[i], (UShort)tteno ); 4246c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 4256c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj} 4266c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 4276c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 4286c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj/* Check the eclass info in 'sec' to ensure it is consistent. Returns 4296c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj True if OK, False if something's not right. Expensive. */ 4306c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 4316c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardjstatic Bool sanity_check_eclasses_in_sector ( Sector* sec ) 4326c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj{ 4336c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj# define BAD(_str) do { whassup = (_str); goto bad; } while (0) 4346c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 4356c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj HChar* whassup = NULL; 4366c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Int i, j, k, n, ec_num, ec_idx; 4376c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj TTEntry* tte; 4386c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj UShort tteno; 4396c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj ULong* tce; 4406c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 4416c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* Basic checks on this sector */ 4426c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (sec->tt_n_inuse < 0 || sec->tt_n_inuse > N_TTES_PER_SECTOR_USABLE) 4436c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj BAD("invalid sec->tt_n_inuse"); 4446c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj tce = sec->tc_next; 4456c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (tce < &sec->tc[0] || tce > &sec->tc[tc_sector_szQ]) 4466c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj BAD("sec->tc_next points outside tc"); 4476c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 4486c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* For each eclass ... */ 4496c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj for (i = 0; i < ECLASS_N; i++) { 4506c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (sec->ec2tte_size[i] == 0 && sec->ec2tte[i] != NULL) 4516c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj BAD("ec2tte_size/ec2tte mismatch(1)"); 4526c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (sec->ec2tte_size[i] != 0 && sec->ec2tte[i] == NULL) 4536c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj BAD("ec2tte_size/ec2tte mismatch(2)"); 4546c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (sec->ec2tte_used[i] < 0 4556c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj || sec->ec2tte_used[i] > sec->ec2tte_size[i]) 4566c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj BAD("implausible ec2tte_used"); 4576c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (sec->ec2tte_used[i] == 0) 4586c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj continue; 4596c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 4606c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* For each tt reference in each eclass .. ensure the reference 4616c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj is to a valid tt entry, and that the entry's address ranges 4626c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj really include this eclass. */ 4636c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 4646c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj for (j = 0; j < sec->ec2tte_used[i]; j++) { 4656c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj tteno = sec->ec2tte[i][j]; 4666c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (tteno == EC2TTE_DELETED) 4676c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj continue; 4686c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (tteno >= N_TTES_PER_SECTOR) 4696c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj BAD("implausible tteno"); 4706c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj tte = &sec->tt[tteno]; 4716c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (tte->status != InUse) 4726c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj BAD("tteno points to non-inuse tte"); 4736c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (tte->n_tte2ec < 1 || tte->n_tte2ec > 3) 4746c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj BAD("tte->n_tte2ec out of range"); 4756c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* Exactly least one of tte->eclasses[0 .. tte->n_eclasses-1] 4766c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj must equal i. Inspect tte's eclass info. */ 4776c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj n = 0; 4786c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj for (k = 0; k < tte->n_tte2ec; k++) { 4796c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (k < tte->n_tte2ec-1 4806c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj && tte->tte2ec_ec[k] >= tte->tte2ec_ec[k+1]) 4816c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj BAD("tte->tte2ec_ec[..] out of order"); 4826c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj ec_num = tte->tte2ec_ec[k]; 4836c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (ec_num < 0 || ec_num >= ECLASS_N) 4846c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj BAD("tte->tte2ec_ec[..] out of range"); 4856c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (ec_num != i) 4866c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj continue; 4876c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj ec_idx = tte->tte2ec_ix[k]; 4886c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (ec_idx < 0 || ec_idx >= sec->ec2tte_used[i]) 4896c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj BAD("tte->tte2ec_ix[..] out of range"); 4906c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (ec_idx == j) 4916c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj n++; 4926c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 4936c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (n != 1) 4946c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj BAD("tteno does not point back at eclass"); 4956c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 4966c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 4976c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 4986c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* That establishes that for each forward pointer from TTEntrys 4996c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj there is a corresponding backward pointer from the eclass[] 5006c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj arrays. However, it doesn't rule out the possibility of other, 5016c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj bogus pointers in the eclass[] arrays. So do those similarly: 5026c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj scan through them and check the TTEntryies they point at point 5036c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj back. */ 5046c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 5056c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj for (i = 0; i < N_TTES_PER_SECTOR_USABLE; i++) { 5066c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 5076c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj tte = &sec->tt[i]; 5086c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (tte->status == Empty || tte->status == Deleted) { 5096c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (tte->n_tte2ec != 0) 5106c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj BAD("tte->n_eclasses nonzero for unused tte"); 5116c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj continue; 5126c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 5136c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 5146c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(tte->status == InUse); 5156c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 5166c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (tte->n_tte2ec < 1 || tte->n_tte2ec > 3) 5176c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj BAD("tte->n_eclasses out of range(2)"); 5186c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 5196c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj for (j = 0; j < tte->n_tte2ec; j++) { 5206c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj ec_num = tte->tte2ec_ec[j]; 5216c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (ec_num < 0 || ec_num >= ECLASS_N) 5226c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj BAD("tte->eclass[..] out of range"); 5236c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj ec_idx = tte->tte2ec_ix[j]; 5246c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (ec_idx < 0 || ec_idx >= sec->ec2tte_used[ec_num]) 5256c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj BAD("tte->ec_idx[..] out of range(2)"); 5266c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (sec->ec2tte[ec_num][ec_idx] != i) 5276c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj BAD("ec2tte does not point back to tte"); 5286c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 5296c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 5306c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 5316c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj return True; 5326c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 5336c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj bad: 5346c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (whassup) 5356c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj VG_(debugLog)(0, "transtab", "eclass sanity fail: %s\n", whassup); 5366c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 5376c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj# if 0 5386c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj VG_(printf)("eclass = %d\n", i); 5396c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj VG_(printf)("tteno = %d\n", (Int)tteno); 5406c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj switch (tte->status) { 5416c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj case InUse: VG_(printf)("InUse\n"); break; 5426c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj case Deleted: VG_(printf)("Deleted\n"); break; 5436c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj case Empty: VG_(printf)("Empty\n"); break; 5446c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 5456c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (tte->status != Empty) { 5466c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj for (k = 0; k < tte->vge.n_used; k++) 5476c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj VG_(printf)("0x%llx %d\n", tte->vge.base[k], 5486c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj (Int)tte->vge.len[k]); 5496c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 5506c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj# endif 5516c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 5526c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj return False; 5536c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 5546c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj# undef BAD 5556c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj} 5566c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 5576c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 5586c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj/* Sanity check absolutely everything. True == check passed. */ 5596c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 5600ec07f32bbbb209d749b9974408e6f025ad40b31sewardj/* forward */ 5610ec07f32bbbb209d749b9974408e6f025ad40b31sewardjstatic Bool sanity_check_redir_tt_tc ( void ); 5620ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 5636c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardjstatic Bool sanity_check_all_sectors ( void ) 5646c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj{ 5656c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Int sno; 5666c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Bool sane; 5676c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Sector* sec; 5686c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj for (sno = 0; sno < N_SECTORS; sno++) { 5696c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sec = §ors[sno]; 5706c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (sec->tc == NULL) 5716c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj continue; 5726c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sane = sanity_check_eclasses_in_sector( sec ); 5736c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (!sane) 5746c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj return False; 5756c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 5760ec07f32bbbb209d749b9974408e6f025ad40b31sewardj if (!sanity_check_redir_tt_tc() ) 5770ec07f32bbbb209d749b9974408e6f025ad40b31sewardj return False; 5786c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj return True; 5796c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj} 5806c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 581fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 582fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/*-------------------------------------------------------------*/ 5836c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj/*--- Add/find translations ---*/ 584fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/*-------------------------------------------------------------*/ 585fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 586fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjstatic UInt vge_osize ( VexGuestExtents* vge ) 587c0d8f68810eb79a5a6cb6049d6a4ab1de7dc4e18sewardj{ 588fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj UInt i, n = 0; 589fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj for (i = 0; i < vge->n_used; i++) 590fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj n += (UInt)vge->len[i]; 591fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj return n; 592c0d8f68810eb79a5a6cb6049d6a4ab1de7dc4e18sewardj} 593c0d8f68810eb79a5a6cb6049d6a4ab1de7dc4e18sewardj 594fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjstatic Bool isValidSector ( Int sector ) 5956c3769f487145a08c01b58d6e5db3ba274062ad4sewardj{ 596fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (sector < 0 || sector >= N_SECTORS) 597fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj return False; 598fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj return True; 5996c3769f487145a08c01b58d6e5db3ba274062ad4sewardj} 600de4a1d01951937632098a6cda45859afa587a06fsewardj 601fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjstatic inline UInt HASH_TT ( Addr64 key ) 602de4a1d01951937632098a6cda45859afa587a06fsewardj{ 603fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj UInt kHi = (UInt)(key >> 32); 604fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj UInt kLo = (UInt)key; 6056c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj UInt k32 = kHi ^ kLo; 6066c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj UInt ror = 7; 6076c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (ror > 0) 6086c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj k32 = (k32 >> ror) | (k32 << (32-ror)); 6096c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj return k32 % N_TTES_PER_SECTOR; 610de4a1d01951937632098a6cda45859afa587a06fsewardj} 611de4a1d01951937632098a6cda45859afa587a06fsewardj 612fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjstatic void setFastCacheEntry ( Addr64 key, ULong* tce, UInt* count ) 613de4a1d01951937632098a6cda45859afa587a06fsewardj{ 6143387dda4479102751d544c176a7bfc24f3766669sewardj UInt cno = (UInt)VG_TT_FAST_HASH(key); 615fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj VG_(tt_fast)[cno] = tce; 616d1c6fa2e77dcffc271175822c9c547bd64febca9sewardj VG_(tt_fastN)[cno] = VG_(clo_profile_flags) > 0 ? count : NULL; 617fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj n_fast_updates++; 6186c3769f487145a08c01b58d6e5db3ba274062ad4sewardj} 619de4a1d01951937632098a6cda45859afa587a06fsewardj 620fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjstatic void invalidateFastCache ( void ) 6216c3769f487145a08c01b58d6e5db3ba274062ad4sewardj{ 622fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj UInt j; 62365e193939481231a8f6699201258e0a2828357e8sewardj /* This loop is popular enough to make it worth unrolling a 62465e193939481231a8f6699201258e0a2828357e8sewardj bit, at least on ppc32. */ 62565e193939481231a8f6699201258e0a2828357e8sewardj vg_assert(VG_TT_FAST_SIZE > 0 && (VG_TT_FAST_SIZE % 4) == 0); 62665e193939481231a8f6699201258e0a2828357e8sewardj for (j = 0; j < VG_TT_FAST_SIZE; j += 4) { 62765e193939481231a8f6699201258e0a2828357e8sewardj VG_(tt_fast)[j+0] = &bogus_tc_entry; 62865e193939481231a8f6699201258e0a2828357e8sewardj VG_(tt_fast)[j+1] = &bogus_tc_entry; 62965e193939481231a8f6699201258e0a2828357e8sewardj VG_(tt_fast)[j+2] = &bogus_tc_entry; 63065e193939481231a8f6699201258e0a2828357e8sewardj VG_(tt_fast)[j+3] = &bogus_tc_entry; 63165e193939481231a8f6699201258e0a2828357e8sewardj VG_(tt_fastN)[j+0] = NULL; 63265e193939481231a8f6699201258e0a2828357e8sewardj VG_(tt_fastN)[j+1] = NULL; 63365e193939481231a8f6699201258e0a2828357e8sewardj VG_(tt_fastN)[j+2] = NULL; 63465e193939481231a8f6699201258e0a2828357e8sewardj VG_(tt_fastN)[j+3] = NULL; 6356c3769f487145a08c01b58d6e5db3ba274062ad4sewardj } 63665e193939481231a8f6699201258e0a2828357e8sewardj vg_assert(j == VG_TT_FAST_SIZE); 637fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj n_fast_flushes++; 6386c3769f487145a08c01b58d6e5db3ba274062ad4sewardj} 639de4a1d01951937632098a6cda45859afa587a06fsewardj 640fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjstatic void initialiseSector ( Int sno ) 6416c3769f487145a08c01b58d6e5db3ba274062ad4sewardj{ 64245f4e7c91119c7d01a59f5e827c67841632c9314sewardj Int i; 64345f4e7c91119c7d01a59f5e827c67841632c9314sewardj SysRes sres; 6446c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Sector* sec; 645fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj vg_assert(isValidSector(sno)); 646fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 6476c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sec = §ors[sno]; 6486c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 6496c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (sec->tc == NULL) { 6506c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 651fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* Sector has never been used before. Need to allocate tt and 652fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj tc. */ 6536c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(sec->tt == NULL); 6546c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(sec->tc_next == NULL); 6556c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(sec->tt_n_inuse == 0); 6566c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj for (i = 0; i < ECLASS_N; i++) { 6576c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(sec->ec2tte_size[i] == 0); 6586c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(sec->ec2tte_used[i] == 0); 6596c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(sec->ec2tte[i] == NULL); 6606c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 66145f4e7c91119c7d01a59f5e827c67841632c9314sewardj 66245f4e7c91119c7d01a59f5e827c67841632c9314sewardj VG_(debugLog)(1,"transtab", "allocate sector %d\n", sno); 66345f4e7c91119c7d01a59f5e827c67841632c9314sewardj 66445f4e7c91119c7d01a59f5e827c67841632c9314sewardj sres = VG_(am_mmap_anon_float_valgrind)( 8 * tc_sector_szQ ); 66545f4e7c91119c7d01a59f5e827c67841632c9314sewardj if (sres.isError) { 66645f4e7c91119c7d01a59f5e827c67841632c9314sewardj VG_(out_of_memory_NORETURN)("initialiseSector(TC)", 66745f4e7c91119c7d01a59f5e827c67841632c9314sewardj 8 * tc_sector_szQ ); 66845f4e7c91119c7d01a59f5e827c67841632c9314sewardj /*NOTREACHED*/ 66945f4e7c91119c7d01a59f5e827c67841632c9314sewardj } 670e808930793aeddc4cfd3e7a94b665913bec2566csewardj sec->tc = (ULong*)sres.res; 67145f4e7c91119c7d01a59f5e827c67841632c9314sewardj 67245f4e7c91119c7d01a59f5e827c67841632c9314sewardj sres = VG_(am_mmap_anon_float_valgrind) 67345f4e7c91119c7d01a59f5e827c67841632c9314sewardj ( N_TTES_PER_SECTOR * sizeof(TTEntry) ); 67445f4e7c91119c7d01a59f5e827c67841632c9314sewardj if (sres.isError) { 67545f4e7c91119c7d01a59f5e827c67841632c9314sewardj VG_(out_of_memory_NORETURN)("initialiseSector(TT)", 67645f4e7c91119c7d01a59f5e827c67841632c9314sewardj N_TTES_PER_SECTOR * sizeof(TTEntry) ); 67745f4e7c91119c7d01a59f5e827c67841632c9314sewardj /*NOTREACHED*/ 67845f4e7c91119c7d01a59f5e827c67841632c9314sewardj } 679e808930793aeddc4cfd3e7a94b665913bec2566csewardj sec->tt = (TTEntry*)sres.res; 6806c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 6816c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj for (i = 0; i < N_TTES_PER_SECTOR; i++) { 6826c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sec->tt[i].status = Empty; 6836c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sec->tt[i].n_tte2ec = 0; 6846c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 68545f4e7c91119c7d01a59f5e827c67841632c9314sewardj 686fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (VG_(clo_verbosity) > 2) 687fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj VG_(message)(Vg_DebugMsg, "TT/TC: initialise sector %d", sno); 6886c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 689fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj } else { 6906c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 6916c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* Sector has been used before. Dump the old contents. */ 69245f4e7c91119c7d01a59f5e827c67841632c9314sewardj VG_(debugLog)(1,"transtab", "recycle sector %d\n", sno); 6936c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(sec->tt != NULL); 6946c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(sec->tc_next != NULL); 6956c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj n_dump_count += sec->tt_n_inuse; 6966c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 6976c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* Visit each just-about-to-be-abandoned translation. */ 698fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj for (i = 0; i < N_TTES_PER_SECTOR; i++) { 6996c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (sec->tt[i].status == InUse) { 7006c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(sec->tt[i].n_tte2ec >= 1); 7016c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(sec->tt[i].n_tte2ec <= 3); 7026c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj n_dump_osize += vge_osize(&sec->tt[i].vge); 7033786772222c5b31fd6cc5586bcd1ce0da58f1dbesewardj /* Tell the tool too. */ 7040b9d74abd0a663b530d290b2b788ddeda46e5400sewardj if (VG_(needs).superblock_discards) { 7050b9d74abd0a663b530d290b2b788ddeda46e5400sewardj VG_TDICT_CALL( tool_discard_superblock_info, 7064ba057cce1d81a949f5a899b5abb99e90a731bccsewardj sec->tt[i].entry, 7076c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sec->tt[i].vge ); 7083786772222c5b31fd6cc5586bcd1ce0da58f1dbesewardj } 7096c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } else { 7106c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(sec->tt[i].n_tte2ec == 0); 7116c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 7126c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sec->tt[i].status = Empty; 7136c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sec->tt[i].n_tte2ec = 0; 7146c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 7156c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 7166c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* Free up the eclass structures. */ 7176c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj for (i = 0; i < ECLASS_N; i++) { 7186c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (sec->ec2tte_size[i] == 0) { 7196c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(sec->ec2tte_used[i] == 0); 7206c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(sec->ec2tte[i] == NULL); 7216c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } else { 7226c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(sec->ec2tte[i] != NULL); 7236c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj VG_(arena_free)(VG_AR_TTAUX, sec->ec2tte[i]); 7246c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sec->ec2tte[i] = NULL; 7256c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sec->ec2tte_size[i] = 0; 7266c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sec->ec2tte_used[i] = 0; 727fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj } 728fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj } 7296c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 730fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (VG_(clo_verbosity) > 2) 731fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj VG_(message)(Vg_DebugMsg, "TT/TC: recycle sector %d", sno); 7326c3769f487145a08c01b58d6e5db3ba274062ad4sewardj } 7334ccf707a1b52898b878fc90dc0054a0c23cc9fccsewardj 7346c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sec->tc_next = sec->tc; 7356c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sec->tt_n_inuse = 0; 736fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 737fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj invalidateFastCache(); 7386c3769f487145a08c01b58d6e5db3ba274062ad4sewardj} 739de4a1d01951937632098a6cda45859afa587a06fsewardj 74010f08cf5b84882eebbb6712a7be890577650e8adsewardjstatic void invalidate_icache ( void *ptr, Int nbytes ) 74185665ca6fa29dd64754dabe50eb98f25896e752acerion{ 7422c48c7b0a453d32375a4df17e153011b797ef28csewardj# if defined(VGA_ppc32) || defined(VGA_ppc64) 74310f08cf5b84882eebbb6712a7be890577650e8adsewardj Addr startaddr = (Addr) ptr; 74410f08cf5b84882eebbb6712a7be890577650e8adsewardj Addr endaddr = startaddr + nbytes; 745e3826cfe34ec9a0caa570a0d15647b28711584a0sewardj Addr cls; 74610f08cf5b84882eebbb6712a7be890577650e8adsewardj Addr addr; 747e3826cfe34ec9a0caa570a0d15647b28711584a0sewardj VexArchInfo vai; 748e3826cfe34ec9a0caa570a0d15647b28711584a0sewardj 749e3826cfe34ec9a0caa570a0d15647b28711584a0sewardj VG_(machine_get_VexArchInfo)( NULL, &vai ); 7501f0d814045aba94e01e62e04e968ca8b970b3d44cerion cls = vai.ppc_cache_line_szB; 75110f08cf5b84882eebbb6712a7be890577650e8adsewardj 7522bf6ba55b04ea9a58098f041c5ee149539c0f081sewardj /* Stay sane .. */ 7532bf6ba55b04ea9a58098f041c5ee149539c0f081sewardj vg_assert(cls == 32 || cls == 128); 75485665ca6fa29dd64754dabe50eb98f25896e752acerion 75585665ca6fa29dd64754dabe50eb98f25896e752acerion startaddr &= ~(cls - 1); 75685665ca6fa29dd64754dabe50eb98f25896e752acerion for (addr = startaddr; addr < endaddr; addr += cls) 75785665ca6fa29dd64754dabe50eb98f25896e752acerion asm volatile("dcbst 0,%0" : : "r" (addr)); 75885665ca6fa29dd64754dabe50eb98f25896e752acerion asm volatile("sync"); 75985665ca6fa29dd64754dabe50eb98f25896e752acerion for (addr = startaddr; addr < endaddr; addr += cls) 76085665ca6fa29dd64754dabe50eb98f25896e752acerion asm volatile("icbi 0,%0" : : "r" (addr)); 76185665ca6fa29dd64754dabe50eb98f25896e752acerion asm volatile("sync; isync"); 76210f08cf5b84882eebbb6712a7be890577650e8adsewardj 76310f08cf5b84882eebbb6712a7be890577650e8adsewardj# elif defined(VGA_x86) 76410f08cf5b84882eebbb6712a7be890577650e8adsewardj /* no need to do anything, hardware provides coherence */ 76510f08cf5b84882eebbb6712a7be890577650e8adsewardj 76610f08cf5b84882eebbb6712a7be890577650e8adsewardj# elif defined(VGA_amd64) 76710f08cf5b84882eebbb6712a7be890577650e8adsewardj /* no need to do anything, hardware provides coherence */ 76810f08cf5b84882eebbb6712a7be890577650e8adsewardj 76910f08cf5b84882eebbb6712a7be890577650e8adsewardj# else 77010f08cf5b84882eebbb6712a7be890577650e8adsewardj# error "Unknown ARCH" 77110f08cf5b84882eebbb6712a7be890577650e8adsewardj# endif 77285665ca6fa29dd64754dabe50eb98f25896e752acerion} 77385665ca6fa29dd64754dabe50eb98f25896e752acerion 774de4a1d01951937632098a6cda45859afa587a06fsewardj 775fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/* Add a translation of vge to TT/TC. The translation is temporarily 776fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj in code[0 .. code_len-1]. 777de4a1d01951937632098a6cda45859afa587a06fsewardj 778fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj pre: youngest_sector points to a valid (although possibly full) 779fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj sector. 780fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj*/ 7818bddf58af8cc7342d4bde6712c5a6a33bf2850d4njnvoid VG_(add_to_transtab)( VexGuestExtents* vge, 7828bddf58af8cc7342d4bde6712c5a6a33bf2850d4njn Addr64 entry, 7838bddf58af8cc7342d4bde6712c5a6a33bf2850d4njn AddrH code, 78426412bd2c4ef3e55683141f132ebd1eb32d8530bsewardj UInt code_len, 78526412bd2c4ef3e55683141f132ebd1eb32d8530bsewardj Bool is_self_checking ) 78622854b907c1925a9af97ad20ca3dacb5e300d6f9sewardj{ 787fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj Int tcAvailQ, reqdQ, y, i; 788fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj ULong *tce, *tce2; 789fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj UChar* srcP; 790fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj UChar* dstP; 791fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 792663a1bd7a8def9d4ef4a10615732a15cc9b09171sewardj vg_assert(init_done); 793fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj vg_assert(vge->n_used >= 1 && vge->n_used <= 3); 794e808930793aeddc4cfd3e7a94b665913bec2566csewardj 795e808930793aeddc4cfd3e7a94b665913bec2566csewardj /* 60000: should agree with N_TMPBUF in m_translate.c. */ 796e808930793aeddc4cfd3e7a94b665913bec2566csewardj vg_assert(code_len > 0 && code_len < 60000); 797fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 798fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (0) 7998bddf58af8cc7342d4bde6712c5a6a33bf2850d4njn VG_(printf)("add_to_transtab(entry = 0x%llx, len = %d)\n", 800fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj entry, code_len); 801fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 802fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj n_in_count++; 803fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj n_in_tsize += code_len; 804fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj n_in_osize += vge_osize(vge); 80526412bd2c4ef3e55683141f132ebd1eb32d8530bsewardj if (is_self_checking) 80626412bd2c4ef3e55683141f132ebd1eb32d8530bsewardj n_in_sc_count++; 807fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 808fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj y = youngest_sector; 809fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj vg_assert(isValidSector(y)); 810fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 811fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (sectors[y].tc == NULL) 812fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj initialiseSector(y); 813fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 814fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* Try putting the translation in this sector. */ 815fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj reqdQ = 1 + ((code_len + 7) >> 3); 816fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 817fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* Will it fit in tc? */ 818fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj tcAvailQ = ((ULong*)(§ors[y].tc[tc_sector_szQ])) 819fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj - ((ULong*)(sectors[y].tc_next)); 820fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj vg_assert(tcAvailQ >= 0); 821fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj vg_assert(tcAvailQ <= tc_sector_szQ); 822fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 823fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (tcAvailQ < reqdQ 824fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj || sectors[y].tt_n_inuse >= N_TTES_PER_SECTOR_USABLE) { 825fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* No. So move on to the next sector. Either it's never been 826fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj used before, in which case it will get its tt/tc allocated 827fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj now, or it has been used before, in which case it is set to be 828fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj empty, hence throwing out the oldest sector. */ 829a16ea0a5fa05dac343e63feddf63f51cb0c340bdsewardj vg_assert(tc_sector_szQ > 0); 830a16ea0a5fa05dac343e63feddf63f51cb0c340bdsewardj VG_(debugLog)(1,"transtab", 831a16ea0a5fa05dac343e63feddf63f51cb0c340bdsewardj "declare sector %d full " 832a16ea0a5fa05dac343e63feddf63f51cb0c340bdsewardj "(TT loading %2d%%, TC loading %2d%%)\n", 833a16ea0a5fa05dac343e63feddf63f51cb0c340bdsewardj y, 834a16ea0a5fa05dac343e63feddf63f51cb0c340bdsewardj (100 * sectors[y].tt_n_inuse) 835a16ea0a5fa05dac343e63feddf63f51cb0c340bdsewardj / N_TTES_PER_SECTOR, 836a16ea0a5fa05dac343e63feddf63f51cb0c340bdsewardj (100 * (tc_sector_szQ - tcAvailQ)) 837a16ea0a5fa05dac343e63feddf63f51cb0c340bdsewardj / tc_sector_szQ); 838fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj youngest_sector++; 839fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (youngest_sector >= N_SECTORS) 840fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj youngest_sector = 0; 841fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj y = youngest_sector; 842fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj initialiseSector(y); 843fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj } 84422854b907c1925a9af97ad20ca3dacb5e300d6f9sewardj 845fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* Be sure ... */ 846fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj tcAvailQ = ((ULong*)(§ors[y].tc[tc_sector_szQ])) 847fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj - ((ULong*)(sectors[y].tc_next)); 848fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj vg_assert(tcAvailQ >= 0); 849fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj vg_assert(tcAvailQ <= tc_sector_szQ); 850fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj vg_assert(tcAvailQ >= reqdQ); 851fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj vg_assert(sectors[y].tt_n_inuse < N_TTES_PER_SECTOR_USABLE); 852fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj vg_assert(sectors[y].tt_n_inuse >= 0); 853fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 854fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* Copy into tc. */ 855fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj tce = sectors[y].tc_next; 856fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj vg_assert(tce >= §ors[y].tc[0]); 857fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj vg_assert(tce <= §ors[y].tc[tc_sector_szQ]); 858fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 859fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj tce[0] = entry; 860fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj dstP = (UChar*)(&tce[1]); 861fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj srcP = (UChar*)code; 862fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj for (i = 0; i < code_len; i++) 863fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj dstP[i] = srcP[i]; 864fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj sectors[y].tc_next += reqdQ; 865fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj sectors[y].tt_n_inuse++; 866fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 86785665ca6fa29dd64754dabe50eb98f25896e752acerion invalidate_icache( dstP, code_len ); 86885665ca6fa29dd64754dabe50eb98f25896e752acerion 869fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* more paranoia */ 870fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj tce2 = sectors[y].tc_next; 871fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj vg_assert(tce2 >= §ors[y].tc[0]); 872fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj vg_assert(tce2 <= §ors[y].tc[tc_sector_szQ]); 873fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 874fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* Find an empty tt slot, and use it. There must be such a slot 875fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj since tt is never allowed to get completely full. */ 876fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj i = HASH_TT(entry); 877fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj vg_assert(i >= 0 && i < N_TTES_PER_SECTOR); 87822854b907c1925a9af97ad20ca3dacb5e300d6f9sewardj while (True) { 879fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (sectors[y].tt[i].status == Empty 880fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj || sectors[y].tt[i].status == Deleted) 881fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj break; 882fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj i++; 883fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (i >= N_TTES_PER_SECTOR) 884fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj i = 0; 88522854b907c1925a9af97ad20ca3dacb5e300d6f9sewardj } 886fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 887fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj sectors[y].tt[i].status = InUse; 888fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj sectors[y].tt[i].tce = tce; 889fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj sectors[y].tt[i].count = 0; 890fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj sectors[y].tt[i].weight = 1; 891fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj sectors[y].tt[i].vge = *vge; 892fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj sectors[y].tt[i].entry = entry; 893fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 8946c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* Update the fast-cache. */ 895fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj setFastCacheEntry( entry, tce, §ors[y].tt[i].count ); 8966c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 8976c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* Note the eclass numbers for this translation. */ 8986c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj upd_eclasses_after_add( §ors[y], i ); 89922854b907c1925a9af97ad20ca3dacb5e300d6f9sewardj} 90022854b907c1925a9af97ad20ca3dacb5e300d6f9sewardj 901fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 902fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/* Search for the translation of the given guest address. If 903fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj requested, a successful search can also cause the fast-caches to be 904fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj updated. 905fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj*/ 906fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjBool VG_(search_transtab) ( /*OUT*/AddrH* result, 907fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj Addr64 guest_addr, 908fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj Bool upd_cache ) 9096c3769f487145a08c01b58d6e5db3ba274062ad4sewardj{ 910fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj Int i, j, k, kstart, sno; 911663a1bd7a8def9d4ef4a10615732a15cc9b09171sewardj 912663a1bd7a8def9d4ef4a10615732a15cc9b09171sewardj vg_assert(init_done); 913fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* Find the initial probe point just once. It will be the same in 914fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj all sectors and avoids multiple expensive % operations. */ 915fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj n_full_lookups++; 916fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj k = -1; 917fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj kstart = HASH_TT(guest_addr); 918fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj vg_assert(kstart >= 0 && kstart < N_TTES_PER_SECTOR); 919fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 920fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* Search in all the sectors. Although the order should not matter, 921fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj it might be most efficient to search in the order youngest to 922fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj oldest. */ 923fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj sno = youngest_sector; 924fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj for (i = 0; i < N_SECTORS; i++) { 925fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 926fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (sectors[sno].tc == NULL) 927fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj goto notfound; /* sector not in use. */ 928fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 929fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj k = kstart; 930fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj for (j = 0; j < N_TTES_PER_SECTOR; j++) { 931fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj n_lookup_probes++; 932fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (sectors[sno].tt[k].status == InUse 933fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj && sectors[sno].tt[k].entry == guest_addr) { 934fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* found it */ 935fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (upd_cache) 936fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj setFastCacheEntry( 937fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj guest_addr, sectors[sno].tt[k].tce, 938fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj §ors[sno].tt[k].count ); 939fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (result) 940fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj *result = sizeof(Addr64) + (AddrH)sectors[sno].tt[k].tce; 941fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj return True; 942fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj } 943fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (sectors[sno].tt[k].status == Empty) 944fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj break; /* not found in this sector */ 945fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj k++; 946fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (k == N_TTES_PER_SECTOR) 947fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj k = 0; 9486c3769f487145a08c01b58d6e5db3ba274062ad4sewardj } 949fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 950fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* If we fall off the end, all entries are InUse and not 951fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj matching, or Deleted. In any case we did not find it in this 952fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj sector. */ 953fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 954fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj notfound: 955fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* move to the next oldest sector */ 956fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj sno = sno==0 ? (N_SECTORS-1) : (sno-1); 9576c3769f487145a08c01b58d6e5db3ba274062ad4sewardj } 958fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 959fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* Not found in any sector. */ 960fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj return False; 9616c3769f487145a08c01b58d6e5db3ba274062ad4sewardj} 962de4a1d01951937632098a6cda45859afa587a06fsewardj 963de4a1d01951937632098a6cda45859afa587a06fsewardj 9646c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj/*-------------------------------------------------------------*/ 9656c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj/*--- Delete translations. ---*/ 9666c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj/*-------------------------------------------------------------*/ 9676c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 9680ec07f32bbbb209d749b9974408e6f025ad40b31sewardj/* forward */ 9690ec07f32bbbb209d749b9974408e6f025ad40b31sewardjstatic void unredir_discard_translations( Addr64, ULong ); 9700ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 9716c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj/* Stuff for deleting translations which intersect with a given 9726c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj address range. Unfortunately, to make this run at a reasonable 9736c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj speed, it is complex. */ 974fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 975fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjstatic inline 976a30545069556d3dca4ca3feb7c621bdcb7b34107sewardjBool overlap1 ( Addr64 s1, ULong r1, Addr64 s2, ULong r2 ) 9776c3769f487145a08c01b58d6e5db3ba274062ad4sewardj{ 978a30545069556d3dca4ca3feb7c621bdcb7b34107sewardj Addr64 e1 = s1 + r1 - 1ULL; 979a30545069556d3dca4ca3feb7c621bdcb7b34107sewardj Addr64 e2 = s2 + r2 - 1ULL; 980fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (e1 < s2 || e2 < s1) 981fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj return False; 982fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj return True; 9836c3769f487145a08c01b58d6e5db3ba274062ad4sewardj} 984de4a1d01951937632098a6cda45859afa587a06fsewardj 985fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjstatic inline 986a30545069556d3dca4ca3feb7c621bdcb7b34107sewardjBool overlaps ( Addr64 start, ULong range, VexGuestExtents* vge ) 9876c3769f487145a08c01b58d6e5db3ba274062ad4sewardj{ 988fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (overlap1(start, range, vge->base[0], (UInt)vge->len[0])) 989fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj return True; 990fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (vge->n_used < 2) 991fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj return False; 992fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (overlap1(start, range, vge->base[1], (UInt)vge->len[1])) 993fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj return True; 994fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (vge->n_used < 3) 995fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj return False; 996fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (overlap1(start, range, vge->base[2], (UInt)vge->len[2])) 997fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj return True; 998fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj return False; 9996c3769f487145a08c01b58d6e5db3ba274062ad4sewardj} 1000de4a1d01951937632098a6cda45859afa587a06fsewardj 1001de4a1d01951937632098a6cda45859afa587a06fsewardj 10026c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj/* Delete a tt entry, and update all the eclass data accordingly. */ 10036c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 10046c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardjstatic void delete_tte ( /*MOD*/Sector* sec, Int tteno ) 10056c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj{ 10066c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Int i, ec_num, ec_idx; 10076c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj TTEntry* tte; 10086c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 10096c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(tteno >= 0 && tteno < N_TTES_PER_SECTOR); 10106c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj tte = &sec->tt[tteno]; 10116c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(tte->status == InUse); 10126c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(tte->n_tte2ec >= 1 && tte->n_tte2ec <= 3); 10136c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 10146c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* Deal with the ec-to-tte links first. */ 10156c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj for (i = 0; i < tte->n_tte2ec; i++) { 10166c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj ec_num = (Int)tte->tte2ec_ec[i]; 10176c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj ec_idx = tte->tte2ec_ix[i]; 10186c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(ec_num >= 0 && ec_num < ECLASS_N); 10196c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(ec_idx >= 0); 10206c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(ec_idx < sec->ec2tte_used[ec_num]); 10216c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* Assert that the two links point at each other. */ 10226c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(sec->ec2tte[ec_num][ec_idx] == (UShort)tteno); 10236c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* "delete" the pointer back to here. */ 10246c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sec->ec2tte[ec_num][ec_idx] = EC2TTE_DELETED; 10256c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 10266c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 10276c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* Now fix up this TTEntry. */ 10286c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj tte->status = Deleted; 10296c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj tte->n_tte2ec = 0; 10306c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 10316c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* Stats .. */ 10326c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sec->tt_n_inuse--; 10336c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj n_disc_count++; 10346c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj n_disc_osize += vge_osize(&tte->vge); 10356c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 10366c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* Tell the tool too. */ 10370b9d74abd0a663b530d290b2b788ddeda46e5400sewardj if (VG_(needs).superblock_discards) { 10380b9d74abd0a663b530d290b2b788ddeda46e5400sewardj VG_TDICT_CALL( tool_discard_superblock_info, 10394ba057cce1d81a949f5a899b5abb99e90a731bccsewardj tte->entry, 10406c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj tte->vge ); 10416c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 10426c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj} 10436c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 10446c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 10456c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj/* Delete translations from sec which intersect specified range, but 10466c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj only consider translations in the specified eclass. */ 10476c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 10486c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardjstatic 10496c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardjBool delete_translations_in_sector_eclass ( /*MOD*/Sector* sec, 10506c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Addr64 guest_start, ULong range, 10516c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Int ec ) 10526c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj{ 10536c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Int i; 10546c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj UShort tteno; 10556c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Bool anyDeld = False; 10566c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj TTEntry* tte; 10576c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 10586c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(ec >= 0 && ec < ECLASS_N); 10596c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 10606c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj for (i = 0; i < sec->ec2tte_used[ec]; i++) { 10616c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 10626c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj tteno = sec->ec2tte[ec][i]; 10636c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (tteno == EC2TTE_DELETED) { 10646c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* already deleted */ 10656c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj continue; 10666c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 10676c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 10686c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(tteno < N_TTES_PER_SECTOR); 10696c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 10706c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj tte = &sec->tt[tteno]; 10716c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(tte->status == InUse); 10726c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 10736c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (overlaps( guest_start, range, &tte->vge )) { 10746c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj anyDeld = True; 10756c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj delete_tte( sec, (Int)tteno ); 10766c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 10776c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 10786c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 10796c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 10806c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj return anyDeld; 10816c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj} 10826c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 10836c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 10846c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj/* Delete translations from sec which intersect specified range, the 10856c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj slow way, by inspecting all translations in sec. */ 10866c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 10876c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardjstatic 10886c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardjBool delete_translations_in_sector ( /*MOD*/Sector* sec, 10896c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Addr64 guest_start, ULong range ) 10906c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj{ 10916c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Int i; 10926c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Bool anyDeld = False; 10936c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 10946c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj for (i = 0; i < N_TTES_PER_SECTOR; i++) { 10956c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (sec->tt[i].status == InUse 10966c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj && overlaps( guest_start, range, &sec->tt[i].vge )) { 10976c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj anyDeld = True; 10986c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj delete_tte( sec, i ); 10996c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 11006c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 11016c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 11026c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj return anyDeld; 11036c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj} 11046c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 11056c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 110645f4e7c91119c7d01a59f5e827c67841632c9314sewardjvoid VG_(discard_translations) ( Addr64 guest_start, ULong range, 110745f4e7c91119c7d01a59f5e827c67841632c9314sewardj HChar* who ) 11086c3769f487145a08c01b58d6e5db3ba274062ad4sewardj{ 11096c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Sector* sec; 11106c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Int sno, ec; 11116c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Bool anyDeleted = False; 11126c3769f487145a08c01b58d6e5db3ba274062ad4sewardj 1113663a1bd7a8def9d4ef4a10615732a15cc9b09171sewardj vg_assert(init_done); 1114663a1bd7a8def9d4ef4a10615732a15cc9b09171sewardj 1115a16ea0a5fa05dac343e63feddf63f51cb0c340bdsewardj VG_(debugLog)(2, "transtab", 111645f4e7c91119c7d01a59f5e827c67841632c9314sewardj "discard_translations(0x%llx, %lld) req by %s\n", 111745f4e7c91119c7d01a59f5e827c67841632c9314sewardj guest_start, range, who ); 111845f4e7c91119c7d01a59f5e827c67841632c9314sewardj 11196c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* Pre-deletion sanity check */ 11206c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (VG_(clo_sanity_level >= 4)) { 11216c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Bool sane = sanity_check_all_sectors(); 11226c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(sane); 11236c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 11246c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 11256c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (range == 0) 11266c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj return; 11276c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 11286c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* There are two different ways to do this. 11296c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 11306c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj If the range fits within a single address-range equivalence 11316c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj class, as will be the case for a cache line sized invalidation, 11326c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj then we only have to inspect the set of translations listed in 11336c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj that equivalence class, and also in the "sin-bin" equivalence 11346c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj class ECLASS_MISC. 11356c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 11366c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Otherwise, the invalidation is of a larger range and probably 11376c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj results from munmap. In this case it's (probably!) faster just 11386c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj to inspect all translations, dump those we don't want, and 11396c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj regenerate the equivalence class information (since modifying it 11406c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj in-situ is even more expensive). 11416c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj */ 11426c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 11436c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* First off, figure out if the range falls within a single class, 11446c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj and if so which one. */ 11456c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 11466c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj ec = ECLASS_MISC; 11476c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (range < (1ULL << ECLASS_SHIFT)) 11486c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj ec = range_to_eclass( guest_start, (UInt)range ); 11496c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 11506c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* if ec is ECLASS_MISC then we aren't looking at just a single 11516c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj class, so use the slow scheme. Else use the fast scheme, 11526c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj examining 'ec' and ECLASS_MISC. */ 11536c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 11546c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (ec != ECLASS_MISC) { 11556c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 11566c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj VG_(debugLog)(2, "transtab", 11576c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj " FAST, ec = %d\n", ec); 11586c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 11596c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* Fast scheme */ 11606c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(ec >= 0 && ec < ECLASS_MISC); 11616c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 11626c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj for (sno = 0; sno < N_SECTORS; sno++) { 11636c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sec = §ors[sno]; 11646c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (sec->tc == NULL) 11656c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj continue; 11666c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj anyDeleted |= delete_translations_in_sector_eclass( 11676c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sec, guest_start, range, ec ); 11686c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj anyDeleted |= delete_translations_in_sector_eclass( 11696c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sec, guest_start, range, ECLASS_MISC ); 11706c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 11716c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 11726c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } else { 11736c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 11746c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* slow scheme */ 11756c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 11766c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj VG_(debugLog)(2, "transtab", 11776c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj " SLOW, ec = %d\n", ec); 11786c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 11796c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj for (sno = 0; sno < N_SECTORS; sno++) { 11806c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sec = §ors[sno]; 11816c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (sec->tc == NULL) 11826c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj continue; 11836c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj anyDeleted |= delete_translations_in_sector( 11846c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sec, guest_start, range ); 11856c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 11866c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 1187de4a1d01951937632098a6cda45859afa587a06fsewardj } 1188de4a1d01951937632098a6cda45859afa587a06fsewardj 1189fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (anyDeleted) 1190fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj invalidateFastCache(); 11916c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 11920ec07f32bbbb209d749b9974408e6f025ad40b31sewardj /* don't forget the no-redir cache */ 11930ec07f32bbbb209d749b9974408e6f025ad40b31sewardj unredir_discard_translations( guest_start, range ); 11940ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 11956c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* Post-deletion sanity check */ 11966c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (VG_(clo_sanity_level >= 4)) { 11976c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Int i; 11986c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj TTEntry* tte; 11996c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Bool sane = sanity_check_all_sectors(); 12006c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(sane); 12016c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj /* But now, also check the requested address range isn't 12026c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj present anywhere. */ 12036c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj for (sno = 0; sno < N_SECTORS; sno++) { 12046c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sec = §ors[sno]; 12056c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (sec->tc == NULL) 12066c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj continue; 12076c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj for (i = 0; i < N_TTES_PER_SECTOR; i++) { 12086c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj tte = &sec->tt[i]; 12096c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (tte->status != InUse) 12106c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj continue; 12116c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj vg_assert(!overlaps( guest_start, range, &tte->vge )); 12126c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 12136c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 12146c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 1215fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj} 12166c3769f487145a08c01b58d6e5db3ba274062ad4sewardj 12176c3769f487145a08c01b58d6e5db3ba274062ad4sewardj 1218fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/*------------------------------------------------------------*/ 12190ec07f32bbbb209d749b9974408e6f025ad40b31sewardj/*--- AUXILIARY: the unredirected TT/TC ---*/ 12200ec07f32bbbb209d749b9974408e6f025ad40b31sewardj/*------------------------------------------------------------*/ 12210ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 12220ec07f32bbbb209d749b9974408e6f025ad40b31sewardj/* A very simple translation cache which holds a small number of 12230ec07f32bbbb209d749b9974408e6f025ad40b31sewardj unredirected translations. This is completely independent of the 12240ec07f32bbbb209d749b9974408e6f025ad40b31sewardj main tt/tc structures. When unredir_tc or unredir_tt becomes full, 12250ec07f32bbbb209d749b9974408e6f025ad40b31sewardj both structures are simply dumped and we start over. 12260ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 12270ec07f32bbbb209d749b9974408e6f025ad40b31sewardj Since these translations are unredirected, the search key is (by 12280ec07f32bbbb209d749b9974408e6f025ad40b31sewardj definition) the first address entry in the .vge field. */ 12290ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 12300ec07f32bbbb209d749b9974408e6f025ad40b31sewardj/* Sized to hold 500 translations of average size 1000 bytes. */ 12310ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 12320ec07f32bbbb209d749b9974408e6f025ad40b31sewardj#define UNREDIR_SZB 1000 12330ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 12340ec07f32bbbb209d749b9974408e6f025ad40b31sewardj#define N_UNREDIR_TT 500 12350ec07f32bbbb209d749b9974408e6f025ad40b31sewardj#define N_UNREDIR_TCQ (N_UNREDIR_TT * UNREDIR_SZB / sizeof(ULong)) 12360ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 12370ec07f32bbbb209d749b9974408e6f025ad40b31sewardjtypedef 12380ec07f32bbbb209d749b9974408e6f025ad40b31sewardj struct { 12390ec07f32bbbb209d749b9974408e6f025ad40b31sewardj VexGuestExtents vge; 12400ec07f32bbbb209d749b9974408e6f025ad40b31sewardj Addr hcode; 12410ec07f32bbbb209d749b9974408e6f025ad40b31sewardj Bool inUse; 12420ec07f32bbbb209d749b9974408e6f025ad40b31sewardj } 12430ec07f32bbbb209d749b9974408e6f025ad40b31sewardj UTCEntry; 12440ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 12450ec07f32bbbb209d749b9974408e6f025ad40b31sewardj/* We just allocate forwards in _tc, never deleting. */ 124678c0c09d429c95115e826ef769ecaa6cff2ac338tomstatic ULong *unredir_tc; 124778c0c09d429c95115e826ef769ecaa6cff2ac338tomstatic Int unredir_tc_used = N_UNREDIR_TCQ; 12480ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 12490ec07f32bbbb209d749b9974408e6f025ad40b31sewardj/* Slots in _tt can come into use and out again (.inUse). 12500ec07f32bbbb209d749b9974408e6f025ad40b31sewardj Nevertheless _tt_highwater is maintained so that invalidations 12510ec07f32bbbb209d749b9974408e6f025ad40b31sewardj don't have to scan all the slots when only a few are in use. 12520ec07f32bbbb209d749b9974408e6f025ad40b31sewardj _tt_highwater holds the index of the highest ever allocated 12530ec07f32bbbb209d749b9974408e6f025ad40b31sewardj slot. */ 12540ec07f32bbbb209d749b9974408e6f025ad40b31sewardjstatic UTCEntry unredir_tt[N_UNREDIR_TT]; 12550ec07f32bbbb209d749b9974408e6f025ad40b31sewardjstatic Int unredir_tt_highwater; 12560ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 12570ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 12580ec07f32bbbb209d749b9974408e6f025ad40b31sewardjstatic void init_unredir_tt_tc ( void ) 12590ec07f32bbbb209d749b9974408e6f025ad40b31sewardj{ 12600ec07f32bbbb209d749b9974408e6f025ad40b31sewardj Int i; 126178c0c09d429c95115e826ef769ecaa6cff2ac338tom if (unredir_tc == NULL) { 126278c0c09d429c95115e826ef769ecaa6cff2ac338tom SysRes sres = VG_(am_mmap_anon_float_valgrind)( N_UNREDIR_TT * UNREDIR_SZB ); 126378c0c09d429c95115e826ef769ecaa6cff2ac338tom if (sres.isError) { 126478c0c09d429c95115e826ef769ecaa6cff2ac338tom VG_(out_of_memory_NORETURN)("init_unredir_tt_tc", N_UNREDIR_TT * UNREDIR_SZB); 126578c0c09d429c95115e826ef769ecaa6cff2ac338tom /*NOTREACHED*/ 126678c0c09d429c95115e826ef769ecaa6cff2ac338tom } 1267e808930793aeddc4cfd3e7a94b665913bec2566csewardj unredir_tc = (ULong *)sres.res; 126878c0c09d429c95115e826ef769ecaa6cff2ac338tom } 12690ec07f32bbbb209d749b9974408e6f025ad40b31sewardj unredir_tc_used = 0; 12700ec07f32bbbb209d749b9974408e6f025ad40b31sewardj for (i = 0; i < N_UNREDIR_TT; i++) 12710ec07f32bbbb209d749b9974408e6f025ad40b31sewardj unredir_tt[i].inUse = False; 12720ec07f32bbbb209d749b9974408e6f025ad40b31sewardj unredir_tt_highwater = -1; 12730ec07f32bbbb209d749b9974408e6f025ad40b31sewardj} 12740ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 12750ec07f32bbbb209d749b9974408e6f025ad40b31sewardj/* Do a sanity check; return False on failure. */ 12760ec07f32bbbb209d749b9974408e6f025ad40b31sewardjstatic Bool sanity_check_redir_tt_tc ( void ) 12770ec07f32bbbb209d749b9974408e6f025ad40b31sewardj{ 12780ec07f32bbbb209d749b9974408e6f025ad40b31sewardj Int i; 12790ec07f32bbbb209d749b9974408e6f025ad40b31sewardj if (unredir_tt_highwater < -1) return False; 12800ec07f32bbbb209d749b9974408e6f025ad40b31sewardj if (unredir_tt_highwater >= N_UNREDIR_TT) return False; 12810ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 12820ec07f32bbbb209d749b9974408e6f025ad40b31sewardj for (i = unredir_tt_highwater+1; i < N_UNREDIR_TT; i++) 12830ec07f32bbbb209d749b9974408e6f025ad40b31sewardj if (unredir_tt[i].inUse) 12840ec07f32bbbb209d749b9974408e6f025ad40b31sewardj return False; 12850ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 12860ec07f32bbbb209d749b9974408e6f025ad40b31sewardj if (unredir_tc_used < 0) return False; 12870ec07f32bbbb209d749b9974408e6f025ad40b31sewardj if (unredir_tc_used > N_UNREDIR_TCQ) return False; 12880ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 12890ec07f32bbbb209d749b9974408e6f025ad40b31sewardj return True; 12900ec07f32bbbb209d749b9974408e6f025ad40b31sewardj} 12910ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 12920ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 12930ec07f32bbbb209d749b9974408e6f025ad40b31sewardj/* Add an UNREDIRECTED translation of vge to TT/TC. The translation 12940ec07f32bbbb209d749b9974408e6f025ad40b31sewardj is temporarily in code[0 .. code_len-1]. 12950ec07f32bbbb209d749b9974408e6f025ad40b31sewardj*/ 12960ec07f32bbbb209d749b9974408e6f025ad40b31sewardjvoid VG_(add_to_unredir_transtab)( VexGuestExtents* vge, 12970ec07f32bbbb209d749b9974408e6f025ad40b31sewardj Addr64 entry, 12980ec07f32bbbb209d749b9974408e6f025ad40b31sewardj AddrH code, 12990ec07f32bbbb209d749b9974408e6f025ad40b31sewardj UInt code_len, 13000ec07f32bbbb209d749b9974408e6f025ad40b31sewardj Bool is_self_checking ) 13010ec07f32bbbb209d749b9974408e6f025ad40b31sewardj{ 13020ec07f32bbbb209d749b9974408e6f025ad40b31sewardj Int i, j, code_szQ; 13030ec07f32bbbb209d749b9974408e6f025ad40b31sewardj HChar *srcP, *dstP; 13040ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 13050ec07f32bbbb209d749b9974408e6f025ad40b31sewardj vg_assert(sanity_check_redir_tt_tc()); 13060ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 13070ec07f32bbbb209d749b9974408e6f025ad40b31sewardj /* This is the whole point: it's not redirected! */ 13080ec07f32bbbb209d749b9974408e6f025ad40b31sewardj vg_assert(entry == vge->base[0]); 13090ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 13100ec07f32bbbb209d749b9974408e6f025ad40b31sewardj /* How many unredir_tt slots are needed */ 13110ec07f32bbbb209d749b9974408e6f025ad40b31sewardj code_szQ = (code_len + 7) / 8; 13120ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 13130ec07f32bbbb209d749b9974408e6f025ad40b31sewardj /* Look for an empty unredir_tc slot */ 13140ec07f32bbbb209d749b9974408e6f025ad40b31sewardj for (i = 0; i < N_UNREDIR_TT; i++) 13150ec07f32bbbb209d749b9974408e6f025ad40b31sewardj if (!unredir_tt[i].inUse) 13160ec07f32bbbb209d749b9974408e6f025ad40b31sewardj break; 13170ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 13180ec07f32bbbb209d749b9974408e6f025ad40b31sewardj if (i >= N_UNREDIR_TT || code_szQ > (N_UNREDIR_TCQ - unredir_tc_used)) { 13190ec07f32bbbb209d749b9974408e6f025ad40b31sewardj /* It's full; dump everything we currently have */ 13200ec07f32bbbb209d749b9974408e6f025ad40b31sewardj init_unredir_tt_tc(); 13210ec07f32bbbb209d749b9974408e6f025ad40b31sewardj i = 0; 13220ec07f32bbbb209d749b9974408e6f025ad40b31sewardj } 13230ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 13240ec07f32bbbb209d749b9974408e6f025ad40b31sewardj vg_assert(unredir_tc_used >= 0); 13250ec07f32bbbb209d749b9974408e6f025ad40b31sewardj vg_assert(unredir_tc_used <= N_UNREDIR_TCQ); 13260ec07f32bbbb209d749b9974408e6f025ad40b31sewardj vg_assert(code_szQ > 0); 13270ec07f32bbbb209d749b9974408e6f025ad40b31sewardj vg_assert(code_szQ + unredir_tc_used <= N_UNREDIR_TCQ); 13280ec07f32bbbb209d749b9974408e6f025ad40b31sewardj vg_assert(i >= 0 && i < N_UNREDIR_TT); 13290ec07f32bbbb209d749b9974408e6f025ad40b31sewardj vg_assert(unredir_tt[i].inUse == False); 13300ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 13310ec07f32bbbb209d749b9974408e6f025ad40b31sewardj if (i > unredir_tt_highwater) 13320ec07f32bbbb209d749b9974408e6f025ad40b31sewardj unredir_tt_highwater = i; 13330ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 13340ec07f32bbbb209d749b9974408e6f025ad40b31sewardj dstP = (HChar*)&unredir_tc[unredir_tc_used]; 13350ec07f32bbbb209d749b9974408e6f025ad40b31sewardj srcP = (HChar*)code; 13360ec07f32bbbb209d749b9974408e6f025ad40b31sewardj for (j = 0; j < code_len; j++) 13370ec07f32bbbb209d749b9974408e6f025ad40b31sewardj dstP[j] = srcP[j]; 13380ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 1339c0a02f88a9b5ca7cebebf67e1f2f5ebcadb1e238sewardj invalidate_icache( dstP, code_len ); 1340c0a02f88a9b5ca7cebebf67e1f2f5ebcadb1e238sewardj 13410ec07f32bbbb209d749b9974408e6f025ad40b31sewardj unredir_tt[i].inUse = True; 13420ec07f32bbbb209d749b9974408e6f025ad40b31sewardj unredir_tt[i].vge = *vge; 13430ec07f32bbbb209d749b9974408e6f025ad40b31sewardj unredir_tt[i].hcode = (Addr)dstP; 13440ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 13450ec07f32bbbb209d749b9974408e6f025ad40b31sewardj unredir_tc_used += code_szQ; 13460ec07f32bbbb209d749b9974408e6f025ad40b31sewardj vg_assert(unredir_tc_used >= 0); 13470ec07f32bbbb209d749b9974408e6f025ad40b31sewardj vg_assert(unredir_tc_used <= N_UNREDIR_TCQ); 13480ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 13490ec07f32bbbb209d749b9974408e6f025ad40b31sewardj vg_assert(&dstP[code_len] <= (HChar*)&unredir_tc[unredir_tc_used]); 13500ec07f32bbbb209d749b9974408e6f025ad40b31sewardj} 13510ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 13520ec07f32bbbb209d749b9974408e6f025ad40b31sewardjBool VG_(search_unredir_transtab) ( /*OUT*/AddrH* result, 13530ec07f32bbbb209d749b9974408e6f025ad40b31sewardj Addr64 guest_addr ) 13540ec07f32bbbb209d749b9974408e6f025ad40b31sewardj{ 13550ec07f32bbbb209d749b9974408e6f025ad40b31sewardj Int i; 13560ec07f32bbbb209d749b9974408e6f025ad40b31sewardj for (i = 0; i < N_UNREDIR_TT; i++) { 13570ec07f32bbbb209d749b9974408e6f025ad40b31sewardj if (!unredir_tt[i].inUse) 13580ec07f32bbbb209d749b9974408e6f025ad40b31sewardj continue; 13590ec07f32bbbb209d749b9974408e6f025ad40b31sewardj if (unredir_tt[i].vge.base[0] == guest_addr) { 13600ec07f32bbbb209d749b9974408e6f025ad40b31sewardj *result = (AddrH)unredir_tt[i].hcode; 13610ec07f32bbbb209d749b9974408e6f025ad40b31sewardj return True; 13620ec07f32bbbb209d749b9974408e6f025ad40b31sewardj } 13630ec07f32bbbb209d749b9974408e6f025ad40b31sewardj } 13640ec07f32bbbb209d749b9974408e6f025ad40b31sewardj return False; 13650ec07f32bbbb209d749b9974408e6f025ad40b31sewardj} 13660ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 13670ec07f32bbbb209d749b9974408e6f025ad40b31sewardjstatic void unredir_discard_translations( Addr64 guest_start, ULong range ) 13680ec07f32bbbb209d749b9974408e6f025ad40b31sewardj{ 13690ec07f32bbbb209d749b9974408e6f025ad40b31sewardj Int i; 13700ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 13710ec07f32bbbb209d749b9974408e6f025ad40b31sewardj vg_assert(sanity_check_redir_tt_tc()); 13720ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 13730ec07f32bbbb209d749b9974408e6f025ad40b31sewardj for (i = 0; i <= unredir_tt_highwater; i++) { 13740ec07f32bbbb209d749b9974408e6f025ad40b31sewardj if (unredir_tt[i].inUse 13750ec07f32bbbb209d749b9974408e6f025ad40b31sewardj && overlaps( guest_start, range, &unredir_tt[i].vge)) 13760ec07f32bbbb209d749b9974408e6f025ad40b31sewardj unredir_tt[i].inUse = False; 13770ec07f32bbbb209d749b9974408e6f025ad40b31sewardj } 13780ec07f32bbbb209d749b9974408e6f025ad40b31sewardj} 13790ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 13800ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 13810ec07f32bbbb209d749b9974408e6f025ad40b31sewardj/*------------------------------------------------------------*/ 1382fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/*--- Initialisation. ---*/ 1383fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/*------------------------------------------------------------*/ 1384fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 1385fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjvoid VG_(init_tt_tc) ( void ) 13866c3769f487145a08c01b58d6e5db3ba274062ad4sewardj{ 13876c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Int i, j, avg_codeszQ; 1388de4a1d01951937632098a6cda45859afa587a06fsewardj 1389663a1bd7a8def9d4ef4a10615732a15cc9b09171sewardj vg_assert(!init_done); 1390663a1bd7a8def9d4ef4a10615732a15cc9b09171sewardj init_done = True; 1391663a1bd7a8def9d4ef4a10615732a15cc9b09171sewardj 1392fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* Otherwise lots of things go wrong... */ 1393fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj vg_assert(sizeof(ULong) == 8); 1394fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj vg_assert(sizeof(Addr64) == 8); 1395de4a1d01951937632098a6cda45859afa587a06fsewardj 1396fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (VG_(clo_verbosity) > 2) 1397fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj VG_(message)(Vg_DebugMsg, 1398fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj "TT/TC: VG_(init_tt_tc) " 1399fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj "(startup of code management)"); 1400fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 1401fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* Figure out how big each tc area should be. */ 140243b9a8abb139b86a24457fa3c19b9cb60ca17c3anjn avg_codeszQ = (VG_(details).avg_translation_sizeB + 7) / 8; 140343b9a8abb139b86a24457fa3c19b9cb60ca17c3anjn tc_sector_szQ = N_TTES_PER_SECTOR_USABLE * (1 + avg_codeszQ); 1404fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 1405fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* Ensure the calculated value is not way crazy. */ 1406fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj vg_assert(tc_sector_szQ >= 2 * N_TTES_PER_SECTOR_USABLE); 1407e808930793aeddc4cfd3e7a94b665913bec2566csewardj vg_assert(tc_sector_szQ <= 80 * N_TTES_PER_SECTOR_USABLE); 1408fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 1409fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* Initialise the sectors */ 1410fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj youngest_sector = 0; 1411fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj for (i = 0; i < N_SECTORS; i++) { 1412fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj sectors[i].tc = NULL; 1413fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj sectors[i].tt = NULL; 1414fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj sectors[i].tc_next = NULL; 1415fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj sectors[i].tt_n_inuse = 0; 14166c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj for (j = 0; j < ECLASS_N; j++) { 14176c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sectors[i].ec2tte_size[j] = 0; 14186c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sectors[i].ec2tte_used[j] = 0; 14196c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj sectors[i].ec2tte[j] = NULL; 14206c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 14216c3769f487145a08c01b58d6e5db3ba274062ad4sewardj } 14224ccf707a1b52898b878fc90dc0054a0c23cc9fccsewardj 1423fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* and the fast caches. */ 1424fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj invalidateFastCache(); 1425fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 14260ec07f32bbbb209d749b9974408e6f025ad40b31sewardj /* and the unredir tt/tc */ 14270ec07f32bbbb209d749b9974408e6f025ad40b31sewardj init_unredir_tt_tc(); 14280ec07f32bbbb209d749b9974408e6f025ad40b31sewardj 1429fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (VG_(clo_verbosity) > 2) { 1430fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj VG_(message)(Vg_DebugMsg, 1431fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj "TT/TC: cache: %d sectors of %d bytes each = %d total", 1432fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj N_SECTORS, 8 * tc_sector_szQ, 1433fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj N_SECTORS * 8 * tc_sector_szQ ); 1434fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj VG_(message)(Vg_DebugMsg, 1435fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj "TT/TC: table: %d total entries, max occupancy %d (%d%%)", 1436fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj N_SECTORS * N_TTES_PER_SECTOR, 1437fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj N_SECTORS * N_TTES_PER_SECTOR_USABLE, 1438fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj SECTOR_TT_LIMIT_PERCENT ); 1439fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj } 144045f4e7c91119c7d01a59f5e827c67841632c9314sewardj 144145f4e7c91119c7d01a59f5e827c67841632c9314sewardj VG_(debugLog)(2, "transtab", 144245f4e7c91119c7d01a59f5e827c67841632c9314sewardj "cache: %d sectors of %d bytes each = %d total\n", 144345f4e7c91119c7d01a59f5e827c67841632c9314sewardj N_SECTORS, 8 * tc_sector_szQ, 144445f4e7c91119c7d01a59f5e827c67841632c9314sewardj N_SECTORS * 8 * tc_sector_szQ ); 144545f4e7c91119c7d01a59f5e827c67841632c9314sewardj VG_(debugLog)(2, "transtab", 144645f4e7c91119c7d01a59f5e827c67841632c9314sewardj "table: %d total entries, max occupancy %d (%d%%)\n", 144745f4e7c91119c7d01a59f5e827c67841632c9314sewardj N_SECTORS * N_TTES_PER_SECTOR, 144845f4e7c91119c7d01a59f5e827c67841632c9314sewardj N_SECTORS * N_TTES_PER_SECTOR_USABLE, 144945f4e7c91119c7d01a59f5e827c67841632c9314sewardj SECTOR_TT_LIMIT_PERCENT ); 1450de4a1d01951937632098a6cda45859afa587a06fsewardj} 1451de4a1d01951937632098a6cda45859afa587a06fsewardj 14524ccf707a1b52898b878fc90dc0054a0c23cc9fccsewardj 1453fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/*------------------------------------------------------------*/ 1454fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/*--- Printing out statistics. ---*/ 1455fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/*------------------------------------------------------------*/ 1456fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 1457fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjstatic ULong safe_idiv( ULong a, ULong b ) 145892e7b7f3d93564e293483fcf0f52f2d6f6c6dcc0nethercote{ 145992e7b7f3d93564e293483fcf0f52f2d6f6c6dcc0nethercote return (b == 0 ? 0 : a / b); 146092e7b7f3d93564e293483fcf0f52f2d6f6c6dcc0nethercote} 146192e7b7f3d93564e293483fcf0f52f2d6f6c6dcc0nethercote 1462fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjUInt VG_(get_bbs_translated) ( void ) 1463fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj{ 1464fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj return n_in_count; 1465fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj} 1466fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 1467fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjvoid VG_(print_tt_tc_stats) ( void ) 146892e7b7f3d93564e293483fcf0f52f2d6f6c6dcc0nethercote{ 146992e7b7f3d93564e293483fcf0f52f2d6f6c6dcc0nethercote VG_(message)(Vg_DebugMsg, 14700fd92f4126a029b2c73fd217b337121c38070066njn " tt/tc: %,llu tt lookups requiring %,llu probes", 1471fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj n_full_lookups, n_lookup_probes ); 147292e7b7f3d93564e293483fcf0f52f2d6f6c6dcc0nethercote VG_(message)(Vg_DebugMsg, 14730fd92f4126a029b2c73fd217b337121c38070066njn " tt/tc: %,llu fast-cache updates, %,llu flushes", 1474fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj n_fast_updates, n_fast_flushes ); 1475fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 147692e7b7f3d93564e293483fcf0f52f2d6f6c6dcc0nethercote VG_(message)(Vg_DebugMsg, 14775096a39f831d8f7d9dbdfb43e0de0d8b13d4d950njn " transtab: new %,lld " 14780fd92f4126a029b2c73fd217b337121c38070066njn "(%,llu -> %,llu; ratio %,llu:10) [%,llu scs]", 1479fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj n_in_count, n_in_osize, n_in_tsize, 148026412bd2c4ef3e55683141f132ebd1eb32d8530bsewardj safe_idiv(10*n_in_tsize, n_in_osize), 148126412bd2c4ef3e55683141f132ebd1eb32d8530bsewardj n_in_sc_count); 148292e7b7f3d93564e293483fcf0f52f2d6f6c6dcc0nethercote VG_(message)(Vg_DebugMsg, 14835096a39f831d8f7d9dbdfb43e0de0d8b13d4d950njn " transtab: dumped %,llu (%,llu -> ?" "?)", 1484fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj n_dump_count, n_dump_osize ); 1485fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj VG_(message)(Vg_DebugMsg, 14865096a39f831d8f7d9dbdfb43e0de0d8b13d4d950njn " transtab: discarded %,llu (%,llu -> ?" "?)", 1487fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj n_disc_count, n_disc_osize ); 14886c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj 14896c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (0) { 14906c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj Int i; 14916c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj VG_(printf)("\n"); 14926c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj for (i = 0; i < ECLASS_N; i++) { 14936c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj VG_(printf)(" %4d", sectors[0].ec2tte_used[i]); 14946c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj if (i % 16 == 15) 14956c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj VG_(printf)("\n"); 14966c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 14976c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj VG_(printf)("\n\n"); 14986c1bbbb1799cc34d05e2c1bae06f5533151eaaa3sewardj } 149992e7b7f3d93564e293483fcf0f52f2d6f6c6dcc0nethercote} 1500de4a1d01951937632098a6cda45859afa587a06fsewardj 1501fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/*------------------------------------------------------------*/ 1502fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/*--- Printing out of profiling results. ---*/ 1503fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj/*------------------------------------------------------------*/ 15044ccf707a1b52898b878fc90dc0054a0c23cc9fccsewardj 1505fa8ec113ecff891bdefb13dd361b40a503a992f8sewardjstatic ULong score ( TTEntry* tte ) 1506fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj{ 1507fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj return ((ULong)tte->weight) * ((ULong)tte->count); 1508de4a1d01951937632098a6cda45859afa587a06fsewardj} 1509de4a1d01951937632098a6cda45859afa587a06fsewardj 15102025cf98a3210c175a90fc77cb599cca1643bd68njnULong VG_(get_BB_profile) ( BBProfEntry tops[], UInt n_tops ) 1511de4a1d01951937632098a6cda45859afa587a06fsewardj{ 1512fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj Int sno, i, r, s; 15132025cf98a3210c175a90fc77cb599cca1643bd68njn ULong score_total; 1514fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 1515fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* First, compute the total weighted count, and find the top N 15162025cf98a3210c175a90fc77cb599cca1643bd68njn ttes. tops contains pointers to the most-used n_tops blocks, in 1517fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj descending order (viz, tops[0] is the highest scorer). */ 15182025cf98a3210c175a90fc77cb599cca1643bd68njn for (i = 0; i < n_tops; i++) { 15192025cf98a3210c175a90fc77cb599cca1643bd68njn tops[i].addr = 0; 15202025cf98a3210c175a90fc77cb599cca1643bd68njn tops[i].score = 0; 15212025cf98a3210c175a90fc77cb599cca1643bd68njn } 1522fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 1523fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj score_total = 0; 1524fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj 1525fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj for (sno = 0; sno < N_SECTORS; sno++) { 1526fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (sectors[sno].tc == NULL) 152718d7513cc08bf982711c8a22b70d56af6aa87b33sewardj continue; 1528fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj for (i = 0; i < N_TTES_PER_SECTOR; i++) { 1529fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (sectors[sno].tt[i].status != InUse) 1530fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj continue; 1531fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj score_total += score(§ors[sno].tt[i]); 1532fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* Find the rank for sectors[sno].tt[i]. */ 15332025cf98a3210c175a90fc77cb599cca1643bd68njn r = n_tops-1; 1534fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj while (True) { 1535fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj if (r == -1) 1536fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj break; 15372025cf98a3210c175a90fc77cb599cca1643bd68njn if (tops[r].addr == 0) { 1538fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj r--; 1539fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj continue; 1540fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj } 15412025cf98a3210c175a90fc77cb599cca1643bd68njn if ( score(§ors[sno].tt[i]) > tops[r].score ) { 1542fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj r--; 1543fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj continue; 1544fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj } 1545fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj break; 1546fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj } 1547fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj r++; 15482025cf98a3210c175a90fc77cb599cca1643bd68njn vg_assert(r >= 0 && r <= n_tops); 1549fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj /* This bb should be placed at r, and bbs above it shifted 1550fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj upwards one slot. */ 15512025cf98a3210c175a90fc77cb599cca1643bd68njn if (r < n_tops) { 15522025cf98a3210c175a90fc77cb599cca1643bd68njn for (s = n_tops-1; s > r; s--) 1553fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj tops[s] = tops[s-1]; 15542025cf98a3210c175a90fc77cb599cca1643bd68njn tops[r].addr = sectors[sno].tt[i].entry; 15552025cf98a3210c175a90fc77cb599cca1643bd68njn tops[r].score = score( §ors[sno].tt[i] ); 1556fa8ec113ecff891bdefb13dd361b40a503a992f8sewardj } 1557de4a1d01951937632098a6cda45859afa587a06fsewardj } 1558de4a1d01951937632098a6cda45859afa587a06fsewardj } 1559de4a1d01951937632098a6cda45859afa587a06fsewardj 15602025cf98a3210c175a90fc77cb599cca1643bd68njn return score_total; 1561c0d8f68810eb79a5a6cb6049d6a4ab1de7dc4e18sewardj} 1562c0d8f68810eb79a5a6cb6049d6a4ab1de7dc4e18sewardj 1563de4a1d01951937632098a6cda45859afa587a06fsewardj/*--------------------------------------------------------------------*/ 15648bddf58af8cc7342d4bde6712c5a6a33bf2850d4njn/*--- end ---*/ 1565de4a1d01951937632098a6cda45859afa587a06fsewardj/*--------------------------------------------------------------------*/ 1566