1ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--------------------------------------------------------------------*/ 3ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--- An implementation of malloc/free which doesn't use sbrk. ---*/ 4ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--- m_mallocfree.c ---*/ 5ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--------------------------------------------------------------------*/ 6ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 7ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* 8ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown This file is part of Valgrind, a dynamic binary instrumentation 9ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown framework. 10ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 11436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov Copyright (C) 2000-2013 Julian Seward 12ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown jseward@acm.org 13ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 14ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown This program is free software; you can redistribute it and/or 15ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown modify it under the terms of the GNU General Public License as 16ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown published by the Free Software Foundation; either version 2 of the 17ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown License, or (at your option) any later version. 18ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 19ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown This program is distributed in the hope that it will be useful, but 20ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown WITHOUT ANY WARRANTY; without even the implied warranty of 21ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 22ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown General Public License for more details. 23ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 24ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown You should have received a copy of the GNU General Public License 25ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown along with this program; if not, write to the Free Software 26ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 27ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 02111-1307, USA. 28ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 29ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown The GNU General Public License is contained in the file COPYING. 30ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 31ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 32ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_core_basics.h" 33ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_core_vki.h" 34ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_core_debuglog.h" 35ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_core_libcbase.h" 36ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_core_aspacemgr.h" 37ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_core_libcassert.h" 38ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_core_libcprint.h" 39ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_core_mallocfree.h" 40ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_core_options.h" 41b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#include "pub_core_libcsetjmp.h" // to keep _threadstate.h happy 42ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_core_threadstate.h" // For VG_INVALID_THREADID 43eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov#include "pub_core_gdbserver.h" 44b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#include "pub_core_transtab.h" 45ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#include "pub_core_tooliface.h" 46ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 47436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov#include "pub_core_inner.h" 48663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng#if defined(ENABLE_INNER_CLIENT_REQUEST) 49663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng#include "memcheck/memcheck.h" 50663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng#endif 51ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 52ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// #define DEBUG_MALLOC // turn on heavyweight debugging machinery 53ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// #define VERBOSE_MALLOC // make verbose, esp. in debugging machinery 54ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 55ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Number and total size of blocks in free queue. Used by mallinfo(). */ 56ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownLong VG_(free_queue_volume) = 0; 57ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownLong VG_(free_queue_length) = 0; 58ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 59ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void cc_analyse_alloc_arena ( ArenaId aid ); /* fwds */ 60ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 61ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------------------------------------------------------*/ 62ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--- Main types ---*/ 63ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------------------------------------------------------*/ 64ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 65ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_MALLOC_LISTS 112 // do not change this 66ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 67ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// The amount you can ask for is limited only by sizeof(SizeT)... 68ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define MAX_PSZB (~((SizeT)0x0)) 69ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 70ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Each arena has a sorted array of superblocks, which expands 71ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// dynamically. This is its initial size. 72ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define SBLOCKS_SIZE_INITIAL 50 73ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 74ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef UChar UByte; 75ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 76ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Layout of an in-use block: 77ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 78ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cost center (OPTIONAL) (VG_MIN_MALLOC_SZB bytes, only when h-p enabled) 79ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown this block total szB (sizeof(SizeT) bytes) 80ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown red zone bytes (depends on Arena.rz_szB, but >= sizeof(void*)) 81ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (payload bytes) 82ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown red zone bytes (depends on Arena.rz_szB, but >= sizeof(void*)) 83ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown this block total szB (sizeof(SizeT) bytes) 84ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 85ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Layout of a block on the free list: 86ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 87ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cost center (OPTIONAL) (VG_MIN_MALLOC_SZB bytes, only when h-p enabled) 88ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown this block total szB (sizeof(SizeT) bytes) 89ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown freelist previous ptr (sizeof(void*) bytes) 90ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown excess red zone bytes (if Arena.rz_szB > sizeof(void*)) 91ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (payload bytes) 92ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown excess red zone bytes (if Arena.rz_szB > sizeof(void*)) 93ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown freelist next ptr (sizeof(void*) bytes) 94ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown this block total szB (sizeof(SizeT) bytes) 95ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 96ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Total size in bytes (bszB) and payload size in bytes (pszB) 97ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown are related by: 98ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 99ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown bszB == pszB + 2*sizeof(SizeT) + 2*a->rz_szB 100ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 101ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown when heap profiling is not enabled, and 102ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 103ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown bszB == pszB + 2*sizeof(SizeT) + 2*a->rz_szB + VG_MIN_MALLOC_SZB 104ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 105ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown when it is enabled. It follows that the minimum overhead per heap 106ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown block for arenas used by the core is: 107ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 108ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 32-bit platforms: 2*4 + 2*4 == 16 bytes 109ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 64-bit platforms: 2*8 + 2*8 == 32 bytes 110ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 111ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown when heap profiling is not enabled, and 112ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 113ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 32-bit platforms: 2*4 + 2*4 + 8 == 24 bytes 114ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 64-bit platforms: 2*8 + 2*8 + 16 == 48 bytes 115ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 116ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown when it is enabled. In all cases, extra overhead may be incurred 117ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown when rounding the payload size up to VG_MIN_MALLOC_SZB. 118ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 119ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Furthermore, both size fields in the block have their least-significant 120ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown bit set if the block is not in use, and unset if it is in use. 121ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (The bottom 3 or so bits are always free for this because of alignment.) 122ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown A block size of zero is not possible, because a block always has at 123ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown least two SizeTs and two pointers of overhead. 124ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 125ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Nb: All Block payloads must be VG_MIN_MALLOC_SZB-aligned. This is 126ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown achieved by ensuring that Superblocks are VG_MIN_MALLOC_SZB-aligned 127ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (see newSuperblock() for how), and that the lengths of the following 128ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown things are a multiple of VG_MIN_MALLOC_SZB: 129ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown - Superblock admin section lengths (due to elastic padding) 130ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown - Block admin section (low and high) lengths (due to elastic redzones) 131ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown - Block payload lengths (due to req_pszB rounding up) 132ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 133ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown The heap-profile cost-center field is 8 bytes even on 32 bit 134ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown platforms. This is so as to keep the payload field 8-aligned. On 135ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a 64-bit platform, this cc-field contains a pointer to a const 136ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown HChar*, which is the cost center name. On 32-bit platforms, the 137ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown pointer lives in the lower-addressed half of the field, regardless 138ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown of the endianness of the host. 139ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 140ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef 141ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct { 142ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // No fields are actually used in this struct, because a Block has 143ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // many variable sized fields and so can't be accessed 144ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // meaningfully with normal fields. So we use access functions all 145ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // the time. This struct gives us a type to use, though. Also, we 146ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // make sizeof(Block) 1 byte so that we can do arithmetic with the 147ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // Block* type in increments of 1! 148ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UByte dummy; 149ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 150ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Block; 151ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 152ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// A superblock. 'padding' is never used, it just ensures that if the 153ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// entire Superblock is aligned to VG_MIN_MALLOC_SZB, then payload_bytes[] 154ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// will be too. It can add small amounts of padding unnecessarily -- eg. 155ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// 8-bytes on 32-bit machines with an 8-byte VG_MIN_MALLOC_SZB -- because 156ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// it's too hard to make a constant expression that works perfectly in all 157ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// cases. 158b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// 'unsplittable' is set to NULL if superblock can be splitted, otherwise 159b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// it is set to the address of the superblock. An unsplittable superblock 160b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// will contain only one allocated block. An unsplittable superblock will 161b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// be unmapped when its (only) allocated block is freed. 162b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// The free space at the end of an unsplittable superblock is not used to 163b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// make a free block. Note that this means that an unsplittable superblock can 164b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// have up to slightly less than 1 page of unused bytes at the end of the 165b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// superblock. 166b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// 'unsplittable' is used to avoid quadratic memory usage for linear 167b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// reallocation of big structures 168b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// (see http://bugs.kde.org/show_bug.cgi?id=250101). 169b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// ??? unsplittable replaces 'void *padding2'. Choosed this 170b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// ??? to avoid changing the alignment logic. Maybe something cleaner 171b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// ??? can be done. 172b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// A splittable block can be reclaimed when all its blocks are freed : 173b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// the reclaim of such a block is deferred till either another superblock 174b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// of the same arena can be reclaimed or till a new superblock is needed 175b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// in any arena. 176ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// payload_bytes[] is made a single big Block when the Superblock is 177ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// created, and then can be split and the splittings remerged, but Blocks 178ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// always cover its entire length -- there's never any unused bytes at the 179ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// end, for example. 180ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef 181ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct _Superblock { 182ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT n_payload_bytes; 183b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov struct _Superblock* unsplittable; 184ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UByte padding[ VG_MIN_MALLOC_SZB - 185ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ((sizeof(struct _Superblock*) + sizeof(SizeT)) % 186ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_MIN_MALLOC_SZB) ]; 187ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UByte payload_bytes[0]; 188ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 189ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Superblock; 190ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 191ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// An arena. 'freelist' is a circular, doubly-linked list. 'rz_szB' is 192ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// elastic, in that it can be bigger than asked-for to ensure alignment. 193ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Browntypedef 194ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown struct { 195436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov const HChar* name; 196ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool clientmem; // Allocates in the client address space? 197ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT rz_szB; // Red zone size in bytes 198ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT min_sblock_szB; // Minimum superblock size in bytes 199b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov SizeT min_unsplittable_sblock_szB; 200b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // Minimum unsplittable superblock size in bytes. To be marked as 201b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // unsplittable, a superblock must have a 202b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // size >= min_unsplittable_sblock_szB and cannot be splitted. 203b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // So, to avoid big overhead, superblocks used to provide aligned 204b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // blocks on big alignments are splittable. 205b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // Unsplittable superblocks will be reclaimed when their (only) 206b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // allocated block is freed. 207b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // Smaller size superblocks are splittable and can be reclaimed when all 208b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // their blocks are freed. 209ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Block* freelist[N_MALLOC_LISTS]; 210ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // A dynamically expanding, ordered array of (pointers to) 211ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // superblocks in the arena. If this array is expanded, which 212ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // is rare, the previous space it occupies is simply abandoned. 213ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // To avoid having to get yet another block from m_aspacemgr for 214ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // the first incarnation of this array, the first allocation of 215ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // it is within this struct. If it has to be expanded then the 216ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // new space is acquired from m_aspacemgr as you would expect. 217ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Superblock** sblocks; 218ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT sblocks_size; 219ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT sblocks_used; 220ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Superblock* sblocks_initial[SBLOCKS_SIZE_INITIAL]; 221b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Superblock* deferred_reclaimed_sb; 222436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov 223436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov // VG_(arena_perm_malloc) returns memory from superblocks 224436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov // only used for permanent blocks. No overhead. These superblocks 225436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov // are not stored in sblocks array above. 226436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov Addr perm_malloc_current; // first byte free in perm_malloc sb. 227436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov Addr perm_malloc_limit; // maximum usable byte in perm_malloc sb. 228436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov 229436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov // Stats only 230436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov SizeT stats__perm_bytes_on_loan; 231436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov SizeT stats__perm_blocks; 232436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov 233b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ULong stats__nreclaim_unsplit; 234b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ULong stats__nreclaim_split; 235b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* total # of reclaim executed for unsplittable/splittable superblocks */ 236b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov SizeT stats__bytes_on_loan; 237b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov SizeT stats__bytes_mmaped; 238b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov SizeT stats__bytes_on_loan_max; 239b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ULong stats__tot_blocks; /* total # blocks alloc'd */ 240b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ULong stats__tot_bytes; /* total # bytes alloc'd */ 241b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ULong stats__nsearches; /* total # freelist checks */ 242b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // If profiling, when should the next profile happen at 243b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // (in terms of stats__bytes_on_loan_max) ? 244ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT next_profile_at; 245b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov SizeT stats__bytes_mmaped_max; 246ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 247ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Arena; 248ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 249ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 250ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------------------------------------------------------*/ 251ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--- Low-level functions for working with Blocks. ---*/ 252ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------------------------------------------------------*/ 253ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 254ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define SIZE_T_0x1 ((SizeT)0x1) 255ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 256436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovstatic const char* probably_your_fault = 257ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "This is probably caused by your program erroneously writing past the\n" 258ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "end of a heap block and corrupting heap metadata. If you fix any\n" 259ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "invalid writes reported by Memcheck, this assertion failure will\n" 260ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "probably go away. Please try that before reporting this as a bug.\n"; 261ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 262ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Mark a bszB as in-use, and not in-use, and remove the in-use attribute. 263ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 264ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownSizeT mk_inuse_bszB ( SizeT bszB ) 265ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 266ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert2(bszB != 0, probably_your_fault); 267ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return bszB & (~SIZE_T_0x1); 268ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 269ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 270ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownSizeT mk_free_bszB ( SizeT bszB ) 271ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 272ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert2(bszB != 0, probably_your_fault); 273ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return bszB | SIZE_T_0x1; 274ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 275ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 276ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownSizeT mk_plain_bszB ( SizeT bszB ) 277ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 278ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert2(bszB != 0, probably_your_fault); 279ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return bszB & (~SIZE_T_0x1); 280ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 281ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 282663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng// Forward definition. 283663860b1408516d02ebfcb3a9999a134e6cfb223Ben Chengstatic 284663860b1408516d02ebfcb3a9999a134e6cfb223Ben Chengvoid ensure_mm_init ( ArenaId aid ); 285663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng 286ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// return either 0 or sizeof(ULong) depending on whether or not 287ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// heap profiling is engaged 288b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov#define hp_overhead_szB() set_at_init_hp_overhead_szB 289b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic SizeT set_at_init_hp_overhead_szB = -1000000; 290b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// startup value chosen to very likely cause a problem if used before 291b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// a proper value is given by ensure_mm_init. 292ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 293ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//--------------------------------------------------------------------------- 294ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 295ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Get a block's size as stored, ie with the in-use/free attribute. 296ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 297ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownSizeT get_bszB_as_is ( Block* b ) 298ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 299ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UByte* b2 = (UByte*)b; 300ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT bszB_lo = *(SizeT*)&b2[0 + hp_overhead_szB()]; 301ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT bszB_hi = *(SizeT*)&b2[mk_plain_bszB(bszB_lo) - sizeof(SizeT)]; 302ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert2(bszB_lo == bszB_hi, 303ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "Heap block lo/hi size mismatch: lo = %llu, hi = %llu.\n%s", 304ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (ULong)bszB_lo, (ULong)bszB_hi, probably_your_fault); 305ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return bszB_lo; 306ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 307ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 308ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Get a block's plain size, ie. remove the in-use/free attribute. 309ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 310ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownSizeT get_bszB ( Block* b ) 311ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 312ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return mk_plain_bszB(get_bszB_as_is(b)); 313ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 314ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 315ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Set the size fields of a block. bszB may have the in-use/free attribute. 316ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 317ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid set_bszB ( Block* b, SizeT bszB ) 318ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 319ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UByte* b2 = (UByte*)b; 320ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *(SizeT*)&b2[0 + hp_overhead_szB()] = bszB; 321ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *(SizeT*)&b2[mk_plain_bszB(bszB) - sizeof(SizeT)] = bszB; 322ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 323ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 324ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//--------------------------------------------------------------------------- 325ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 326ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Does this block have the in-use attribute? 327ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 328ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownBool is_inuse_block ( Block* b ) 329ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 330ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT bszB = get_bszB_as_is(b); 331ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert2(bszB != 0, probably_your_fault); 332ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return (0 != (bszB & SIZE_T_0x1)) ? False : True; 333ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 334ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 335ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//--------------------------------------------------------------------------- 336ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 337ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Return the lower, upper and total overhead in bytes for a block. 338ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// These are determined purely by which arena the block lives in. 339ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 340ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownSizeT overhead_szB_lo ( Arena* a ) 341ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 342ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return hp_overhead_szB() + sizeof(SizeT) + a->rz_szB; 343ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 344ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 345ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownSizeT overhead_szB_hi ( Arena* a ) 346ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 347ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return a->rz_szB + sizeof(SizeT); 348ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 349ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 350ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownSizeT overhead_szB ( Arena* a ) 351ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 352ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return overhead_szB_lo(a) + overhead_szB_hi(a); 353ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 354ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 355ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//--------------------------------------------------------------------------- 356ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 357ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Return the minimum bszB for a block in this arena. Can have zero-length 358ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// payloads, so it's the size of the admin bytes. 359ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 360ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownSizeT min_useful_bszB ( Arena* a ) 361ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 362ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return overhead_szB(a); 363ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 364ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 365ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//--------------------------------------------------------------------------- 366ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 367ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Convert payload size <--> block size (both in bytes). 368ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 369ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownSizeT pszB_to_bszB ( Arena* a, SizeT pszB ) 370ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 371ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return pszB + overhead_szB(a); 372ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 373ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 374ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownSizeT bszB_to_pszB ( Arena* a, SizeT bszB ) 375ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 376ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert2(bszB >= overhead_szB(a), probably_your_fault); 377ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return bszB - overhead_szB(a); 378ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 379ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 380ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//--------------------------------------------------------------------------- 381ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 382ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Get a block's payload size. 383ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 384ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownSizeT get_pszB ( Arena* a, Block* b ) 385ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 386ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return bszB_to_pszB(a, get_bszB(b)); 387ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 388ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 389ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//--------------------------------------------------------------------------- 390ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 391ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Given the addr of a block, return the addr of its payload, and vice versa. 392ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 393ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownUByte* get_block_payload ( Arena* a, Block* b ) 394ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 395ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UByte* b2 = (UByte*)b; 396ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return & b2[ overhead_szB_lo(a) ]; 397ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 398ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Given the addr of a block's payload, return the addr of the block itself. 399ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 400ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownBlock* get_payload_block ( Arena* a, UByte* payload ) 401ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 402ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return (Block*)&payload[ -overhead_szB_lo(a) ]; 403ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 404ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 405ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//--------------------------------------------------------------------------- 406ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 407ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Set and get the next and previous link fields of a block. 408ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 409ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid set_prev_b ( Block* b, Block* prev_p ) 410ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 411ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UByte* b2 = (UByte*)b; 412ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *(Block**)&b2[hp_overhead_szB() + sizeof(SizeT)] = prev_p; 413ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 414ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 415ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid set_next_b ( Block* b, Block* next_p ) 416ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 417ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UByte* b2 = (UByte*)b; 418ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown *(Block**)&b2[get_bszB(b) - sizeof(SizeT) - sizeof(void*)] = next_p; 419ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 420ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 421ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownBlock* get_prev_b ( Block* b ) 422ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 423ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UByte* b2 = (UByte*)b; 424ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return *(Block**)&b2[hp_overhead_szB() + sizeof(SizeT)]; 425ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 426ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 427ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownBlock* get_next_b ( Block* b ) 428ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 429ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UByte* b2 = (UByte*)b; 430ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return *(Block**)&b2[get_bszB(b) - sizeof(SizeT) - sizeof(void*)]; 431ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 432ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 433ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//--------------------------------------------------------------------------- 434ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 435ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Set and get the cost-center field of a block. 436ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 437436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovvoid set_cc ( Block* b, const HChar* cc ) 438ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 439ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UByte* b2 = (UByte*)b; 440ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert( VG_(clo_profile_heap) ); 441436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov *(const HChar**)&b2[0] = cc; 442ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 443ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 444436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovconst HChar* get_cc ( Block* b ) 445ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 446ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UByte* b2 = (UByte*)b; 447ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert( VG_(clo_profile_heap) ); 448436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov return *(const HChar**)&b2[0]; 449ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 450ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 451ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//--------------------------------------------------------------------------- 452ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 453ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Get the block immediately preceding this one in the Superblock. 454ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 455ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownBlock* get_predecessor_block ( Block* b ) 456ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 457ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UByte* b2 = (UByte*)b; 458ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT bszB = mk_plain_bszB( (*(SizeT*)&b2[-sizeof(SizeT)]) ); 459ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return (Block*)&b2[-bszB]; 460ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 461ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 462ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown//--------------------------------------------------------------------------- 463ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 464ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Read and write the lower and upper red-zone bytes of a block. 465ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 466ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid set_rz_lo_byte ( Block* b, UInt rz_byteno, UByte v ) 467ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 468ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UByte* b2 = (UByte*)b; 469ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b2[hp_overhead_szB() + sizeof(SizeT) + rz_byteno] = v; 470ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 471ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 472ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid set_rz_hi_byte ( Block* b, UInt rz_byteno, UByte v ) 473ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 474ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UByte* b2 = (UByte*)b; 475ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b2[get_bszB(b) - sizeof(SizeT) - rz_byteno - 1] = v; 476ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 477ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 478ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownUByte get_rz_lo_byte ( Block* b, UInt rz_byteno ) 479ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 480ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UByte* b2 = (UByte*)b; 481ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return b2[hp_overhead_szB() + sizeof(SizeT) + rz_byteno]; 482ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 483ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 484ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownUByte get_rz_hi_byte ( Block* b, UInt rz_byteno ) 485ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 486ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UByte* b2 = (UByte*)b; 487ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return b2[get_bszB(b) - sizeof(SizeT) - rz_byteno - 1]; 488ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 489ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 490ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 491ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------------------------------------------------------*/ 492ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--- Arena management ---*/ 493ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------------------------------------------------------*/ 494ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 495ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define CORE_ARENA_MIN_SZB 1048576 496ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 497ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// The arena structures themselves. 498ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Arena vg_arena[VG_N_ARENAS]; 499ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 500ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Functions external to this module identify arenas using ArenaIds, 501ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// not Arena*s. This fn converts the former to the latter. 502ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Arena* arenaId_to_ArenaP ( ArenaId arena ) 503ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 504ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(arena >= 0 && arena < VG_N_ARENAS); 505ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return & vg_arena[arena]; 506ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 507ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 508436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovstatic ArenaId arenaP_to_ArenaId ( Arena *a ) 509663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng{ 510436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov ArenaId arena = a -vg_arena; 511436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov vg_assert(arena >= 0 && arena < VG_N_ARENAS); 512436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov return arena; 513663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng} 514663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng 515663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng// Initialise an arena. rz_szB is the (default) minimum redzone size; 516663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng// It might be overriden by VG_(clo_redzone_size) or VG_(clo_core_redzone_size). 517663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng// it might be made bigger to ensure that VG_MIN_MALLOC_SZB is observed. 518ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 519436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovvoid arena_init ( ArenaId aid, const HChar* name, SizeT rz_szB, 520b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov SizeT min_sblock_szB, SizeT min_unsplittable_sblock_szB ) 521ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 522ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT i; 523ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Arena* a = arenaId_to_ArenaP(aid); 524663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng 525663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // Ensure default redzones are a reasonable size. 526663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng vg_assert(rz_szB <= MAX_REDZONE_SZB); 527ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 528663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng /* Override the default redzone size if a clo value was given. 529663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng Note that the clo value can be significantly bigger than MAX_REDZONE_SZB 530663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng to allow the user to chase horrible bugs using up to 1 page 531663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng of protection. */ 532663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng if (VG_AR_CLIENT == aid) { 533663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng if (VG_(clo_redzone_size) != -1) 534663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng rz_szB = VG_(clo_redzone_size); 535663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng } else { 536663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng if (VG_(clo_core_redzone_size) != rz_szB) 537663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng rz_szB = VG_(clo_core_redzone_size); 538663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng } 539663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng 540663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // Redzones must always be at least the size of a pointer, for holding the 541663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // prev/next pointer (see the layout details at the top of this file). 542ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (rz_szB < sizeof(void*)) rz_szB = sizeof(void*); 543ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 544ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // The size of the low and high admin sections in a block must be a 545ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // multiple of VG_MIN_MALLOC_SZB. So we round up the asked-for 546ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // redzone size if necessary to achieve this. 547ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a->rz_szB = rz_szB; 548ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (0 != overhead_szB_lo(a) % VG_MIN_MALLOC_SZB) a->rz_szB++; 549ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(overhead_szB_lo(a) - hp_overhead_szB() == overhead_szB_hi(a)); 550ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 551663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // Here we have established the effective redzone size. 552663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng 553663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng 554663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng vg_assert((min_sblock_szB % VKI_PAGE_SIZE) == 0); 555663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng a->name = name; 556663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng a->clientmem = ( VG_AR_CLIENT == aid ? True : False ); 557663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng 558ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a->min_sblock_szB = min_sblock_szB; 559b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->min_unsplittable_sblock_szB = min_unsplittable_sblock_szB; 560ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_MALLOC_LISTS; i++) a->freelist[i] = NULL; 561ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 562b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->sblocks = & a->sblocks_initial[0]; 563b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->sblocks_size = SBLOCKS_SIZE_INITIAL; 564b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->sblocks_used = 0; 565436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov a->deferred_reclaimed_sb = 0; 566436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov a->perm_malloc_current = 0; 567436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov a->perm_malloc_limit = 0; 568436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov a->stats__perm_bytes_on_loan= 0; 569436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov a->stats__perm_blocks = 0; 570b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->stats__nreclaim_unsplit = 0; 571b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->stats__nreclaim_split = 0; 572b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->stats__bytes_on_loan = 0; 573b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->stats__bytes_mmaped = 0; 574b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->stats__bytes_on_loan_max = 0; 575b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->stats__bytes_mmaped_max = 0; 576b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->stats__tot_blocks = 0; 577b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->stats__tot_bytes = 0; 578b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->stats__nsearches = 0; 579b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->next_profile_at = 25 * 1000 * 1000; 580ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(sizeof(a->sblocks_initial) 581ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown == SBLOCKS_SIZE_INITIAL * sizeof(Superblock*)); 582ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 583ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 584ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Print vital stats for an arena. */ 585ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid VG_(print_all_arena_stats) ( void ) 586ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 587ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt i; 588ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < VG_N_ARENAS; i++) { 589ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Arena* a = arenaId_to_ArenaP(i); 590ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(message)(Vg_DebugMsg, 591436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov "%8s: %8lu/%8lu max/curr mmap'd, " 592b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "%llu/%llu unsplit/split sb unmmap'd, " 593436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov "%8lu/%8lu max/curr, " 594b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "%10llu/%10llu totalloc-blocks/bytes," 595663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng " %10llu searches %lu rzB\n", 596b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->name, 597b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->stats__bytes_mmaped_max, a->stats__bytes_mmaped, 598b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->stats__nreclaim_unsplit, a->stats__nreclaim_split, 599b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->stats__bytes_on_loan_max, 600b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->stats__bytes_on_loan, 601b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->stats__tot_blocks, a->stats__tot_bytes, 602663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng a->stats__nsearches, 603663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng a->rz_szB 604ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ); 605ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 606ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 607ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 608ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid VG_(print_arena_cc_analysis) ( void ) 609ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 610ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt i; 611ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert( VG_(clo_profile_heap) ); 612ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < VG_N_ARENAS; i++) { 613ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cc_analyse_alloc_arena(i); 614ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 615ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 616ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 617ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 618ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* This library is self-initialising, as it makes this more self-contained, 619ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown less coupled with the outside world. Hence VG_(arena_malloc)() and 620ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(arena_free)() below always call ensure_mm_init() to ensure things are 621ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown correctly initialised. 622ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 623ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown We initialise the client arena separately (and later) because the core 624ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown must do non-client allocation before the tool has a chance to set the 625ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown client arena's redzone size. 626ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 627ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Bool client_inited = False; 628ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic Bool nonclient_inited = False; 629ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 630ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 631ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid ensure_mm_init ( ArenaId aid ) 632ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 633ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown static SizeT client_rz_szB = 8; // default: be paranoid 634ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 635ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* We use checked red zones (of various sizes) for our internal stuff, 636ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown and an unchecked zone of arbitrary size for the client. Of 637ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown course the client's red zone can be checked by the tool, eg. 638ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown by using addressibility maps, but not by the mechanism implemented 639ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown here, which merely checks at the time of freeing that the red 640ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown zone bytes are unchanged. 641ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 642ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Nb: redzone sizes are *minimums*; they could be made bigger to ensure 643ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown alignment. Eg. with 8 byte alignment, on 32-bit machines 4 stays as 644ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 4, but 16 becomes 20; but on 64-bit machines 4 becomes 8, and 16 645ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown stays as 16 --- the extra 4 bytes in both are accounted for by the 646ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown larger prev/next ptr. 647ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown */ 648ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (VG_AR_CLIENT == aid) { 649ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Int ar_client_sbszB; 650ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (client_inited) { 651ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // This assertion ensures that a tool cannot try to change the client 652ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // redzone size with VG_(needs_malloc_replacement)() after this module 653ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // has done its first allocation from the client arena. 654ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (VG_(needs).malloc_replacement) 655ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(client_rz_szB == VG_(tdict).tool_client_redzone_szB); 656ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 657ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 658ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 659ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // Check and set the client arena redzone size 660ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (VG_(needs).malloc_replacement) { 661ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown client_rz_szB = VG_(tdict).tool_client_redzone_szB; 662663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng if (client_rz_szB > MAX_REDZONE_SZB) { 663ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( "\nTool error:\n" 664ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown " specified redzone size is too big (%llu)\n", 665ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (ULong)client_rz_szB); 666ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(exit)(1); 667ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 668ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 669b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // Initialise the client arena. On all platforms, 670ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // increasing the superblock size reduces the number of superblocks 671ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // in the client arena, which makes findSb cheaper. 672ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ar_client_sbszB = 4194304; 673b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // superblocks with a size > ar_client_sbszB will be unsplittable 674b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // (unless used for providing memalign-ed blocks). 675b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov arena_init ( VG_AR_CLIENT, "client", client_rz_szB, 676b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ar_client_sbszB, ar_client_sbszB+1); 677ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown client_inited = True; 678ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 679ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 680ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (nonclient_inited) { 681ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 682ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 683b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov set_at_init_hp_overhead_szB = 684b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(clo_profile_heap) ? VG_MIN_MALLOC_SZB : 0; 685ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // Initialise the non-client arenas 686b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // Similarly to client arena, big allocations will be unsplittable. 687663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng arena_init ( VG_AR_CORE, "core", CORE_REDZONE_DEFAULT_SZB, 688663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng 4194304, 4194304+1 ); 689663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng arena_init ( VG_AR_DINFO, "dinfo", CORE_REDZONE_DEFAULT_SZB, 690663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng 1048576, 1048576+1 ); 691663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng arena_init ( VG_AR_DEMANGLE, "demangle", CORE_REDZONE_DEFAULT_SZB, 692663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng 65536, 65536+1 ); 693663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng arena_init ( VG_AR_TTAUX, "ttaux", CORE_REDZONE_DEFAULT_SZB, 694663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng 65536, 65536+1 ); 695ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown nonclient_inited = True; 696ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 697ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 698ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# ifdef DEBUG_MALLOC 699ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("ZZZ1\n"); 700ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(sanity_check_malloc_all)(); 701ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("ZZZ2\n"); 702ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# endif 703ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 704ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 705ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 706ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------------------------------------------------------*/ 707ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--- Superblock management ---*/ 708ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------------------------------------------------------*/ 709ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 710ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown__attribute__((noreturn)) 711436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovvoid VG_(out_of_memory_NORETURN) ( const HChar* who, SizeT szB ) 712ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 713436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov static Int outputTrial = 0; 714436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov // We try once to output the full memory state followed by the below message. 715436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov // If that fails (due to out of memory during first trial), we try to just 716436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov // output the below message. 717436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov // And then we abandon. 718436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov 719ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ULong tot_alloc = VG_(am_get_anonsize_total)(); 720436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov const HChar* s1 = 721ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "\n" 722ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown " Valgrind's memory management: out of memory:\n" 723ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown " %s's request for %llu bytes failed.\n" 724ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown " %llu bytes have already been allocated.\n" 725ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown " Valgrind cannot continue. Sorry.\n\n" 726ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown " There are several possible reasons for this.\n" 727ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown " - You have some kind of memory limit in place. Look at the\n" 728ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown " output of 'ulimit -a'. Is there a limit on the size of\n" 729ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown " virtual memory or address space?\n" 730ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown " - You have run out of swap space.\n" 731ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown " - Valgrind has a bug. If you think this is the case or you are\n" 732ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown " not sure, please let us know and we'll try to fix it.\n" 733ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown " Please note that programs can take substantially more memory than\n" 734ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown " normal when running under Valgrind tools, eg. up to twice or\n" 735ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown " more, depending on the tool. On a 64-bit machine, Valgrind\n" 736ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown " should be able to make use of up 32GB memory. On a 32-bit\n" 737ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown " machine, Valgrind should be able to use all the memory available\n" 738ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown " to a single process, up to 4GB if that's how you have your\n" 739ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown " kernel configured. Most 32-bit Linux setups allow a maximum of\n" 740ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown " 3GB per process.\n\n" 741ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown " Whatever the reason, Valgrind cannot continue. Sorry.\n"; 742ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 743436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov if (outputTrial <= 1) { 744436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov if (outputTrial == 0) { 745436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov outputTrial++; 746eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov // First print the memory stats with the aspacemgr data. 747436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov VG_(am_show_nsegments) (0, "out_of_memory"); 748436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov VG_(print_all_arena_stats) (); 749436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov if (VG_(clo_profile_heap)) 750436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov VG_(print_arena_cc_analysis) (); 751eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov // And then print some other information that might help. 752eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov VG_(print_all_stats) (False, /* Memory stats */ 753eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov True /* Tool stats */); 754eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov VG_(show_sched_status) (True, // host_stacktrace 755eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov True, // valgrind_stack_usage 756eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov True); // exited_threads 757eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov /* In case we are an inner valgrind, asks the outer to report 758436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov its memory state in its log output. */ 759436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov INNER_REQUEST(VALGRIND_MONITOR_COMMAND("v.set log_output")); 760436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov INNER_REQUEST(VALGRIND_MONITOR_COMMAND("v.info memory aspacemgr")); 761436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov } 762436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov outputTrial++; 763ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(message)(Vg_UserMsg, s1, who, (ULong)szB, tot_alloc); 764ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 765ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(debugLog)(0,"mallocfree", s1, who, (ULong)szB, tot_alloc); 766ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 767ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 768ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(exit)(1); 769ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 770ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 771ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 772ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Align ptr p upwards to an align-sized boundary. 773ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 774ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid* align_upwards ( void* p, SizeT align ) 775ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 776ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Addr a = (Addr)p; 777ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if ((a % align) == 0) return (void*)a; 778ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return (void*)(a - (a % align) + align); 779ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 780ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 781b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// Forward definition. 782b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic 783b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovvoid deferred_reclaimSuperblock ( Arena* a, Superblock* sb); 784b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 785ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// If not enough memory available, either aborts (for non-client memory) 786ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// or returns 0 (for client memory). 787ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 788ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownSuperblock* newSuperblock ( Arena* a, SizeT cszB ) 789ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 790ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Superblock* sb; 791ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SysRes sres; 792b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Bool unsplittable; 793b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ArenaId aid; 794b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 795b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // A new superblock is needed for arena a. We will execute the deferred 796b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // reclaim in all arenas in order to minimise fragmentation and 797b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // peak memory usage. 798b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for (aid = 0; aid < VG_N_ARENAS; aid++) { 799b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Arena* arena = arenaId_to_ArenaP(aid); 800b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (arena->deferred_reclaimed_sb != NULL) 801b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov deferred_reclaimSuperblock (arena, NULL); 802b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 803ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 804ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // Take into account admin bytes in the Superblock. 805ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cszB += sizeof(Superblock); 806ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 807ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (cszB < a->min_sblock_szB) cszB = a->min_sblock_szB; 808ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cszB = VG_PGROUNDUP(cszB); 809663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng 810b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (cszB >= a->min_unsplittable_sblock_szB) 811b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov unsplittable = True; 812b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov else 813b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov unsplittable = False; 814b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 815ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 816ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (a->clientmem) { 817ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // client allocation -- return 0 to client if it fails 818eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov sres = VG_(am_mmap_anon_float_client) 819eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov ( cszB, VKI_PROT_READ|VKI_PROT_WRITE|VKI_PROT_EXEC ); 820ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (sr_isError(sres)) 821ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0; 822ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sb = (Superblock*)(AddrH)sr_Res(sres); 823ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // Mark this segment as containing client heap. The leak 824ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // checker needs to be able to identify such segments so as not 825ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // to use them as sources of roots during leak checks. 826436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov VG_(am_set_segment_isCH_if_SkAnonC)( VG_(am_find_nsegment)( (Addr)sb ) ); 827ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 828ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // non-client allocation -- abort if it fails 829eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov sres = VG_(am_mmap_anon_float_valgrind)( cszB ); 830ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (sr_isError(sres)) { 831ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(out_of_memory_NORETURN)("newSuperblock", cszB); 832ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* NOTREACHED */ 833ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sb = NULL; /* keep gcc happy */ 834ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 835ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sb = (Superblock*)(AddrH)sr_Res(sres); 836ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 837ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 838ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(NULL != sb); 839663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng INNER_REQUEST(VALGRIND_MAKE_MEM_UNDEFINED(sb, cszB)); 840ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(0 == (Addr)sb % VG_MIN_MALLOC_SZB); 841ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sb->n_payload_bytes = cszB - sizeof(Superblock); 842b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov sb->unsplittable = (unsplittable ? sb : NULL); 843b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->stats__bytes_mmaped += cszB; 844b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (a->stats__bytes_mmaped > a->stats__bytes_mmaped_max) 845b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->stats__bytes_mmaped_max = a->stats__bytes_mmaped; 846ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(debugLog)(1, "mallocfree", 847b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "newSuperblock at %p (pszB %7ld) %s owner %s/%s\n", 848b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov sb, sb->n_payload_bytes, 849b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov (unsplittable ? "unsplittable" : ""), 850ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a->clientmem ? "CLIENT" : "VALGRIND", a->name ); 851ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return sb; 852ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 853ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 854b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// Reclaims the given superblock: 855b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// * removes sb from arena sblocks list. 856b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// * munmap the superblock segment. 857b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic 858b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovvoid reclaimSuperblock ( Arena* a, Superblock* sb) 859b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 860b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov SysRes sres; 861b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov SizeT cszB; 862b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt i, j; 863b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 864b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(debugLog)(1, "mallocfree", 865b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "reclaimSuperblock at %p (pszB %7ld) %s owner %s/%s\n", 866b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov sb, sb->n_payload_bytes, 867b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov (sb->unsplittable ? "unsplittable" : ""), 868b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->clientmem ? "CLIENT" : "VALGRIND", a->name ); 869b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 870b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // Take into account admin bytes in the Superblock. 871b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov cszB = sizeof(Superblock) + sb->n_payload_bytes; 872b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 873b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // removes sb from superblock list. 874b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for (i = 0; i < a->sblocks_used; i++) { 875b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (a->sblocks[i] == sb) 876b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov break; 877b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 878b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov vg_assert(i >= 0 && i < a->sblocks_used); 879b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for (j = i; j < a->sblocks_used; j++) 880b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->sblocks[j] = a->sblocks[j+1]; 881b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->sblocks_used--; 882b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->sblocks[a->sblocks_used] = NULL; 883b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // paranoia: NULLify ptr to reclaimed sb or NULLify copy of ptr to last sb. 884b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 885b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->stats__bytes_mmaped -= cszB; 886b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (sb->unsplittable) 887b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->stats__nreclaim_unsplit++; 888b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov else 889b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->stats__nreclaim_split++; 890b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 891b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // Now that the sb is removed from the list, mnumap its space. 892b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (a->clientmem) { 893b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // reclaimable client allocation 894b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Bool need_discard = False; 895b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov sres = VG_(am_munmap_client)(&need_discard, (Addr) sb, cszB); 896b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov vg_assert2(! sr_isError(sres), "superblock client munmap failure\n"); 897b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* We somewhat help the client by discarding the range. 898b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Note however that if the client has JITted some code in 899b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a small block that was freed, we do not provide this 900b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 'discard support' */ 901b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* JRS 2011-Sept-26: it would be nice to move the discard 902b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov outwards somewhat (in terms of calls) so as to make it easier 903b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov to verify that there will be no nonterminating recursive set 904b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov of calls a result of calling VG_(discard_translations). 905b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Another day, perhaps. */ 906b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (need_discard) 907b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(discard_translations) ((Addr) sb, cszB, "reclaimSuperblock"); 908b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } else { 909b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // reclaimable non-client allocation 910b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov sres = VG_(am_munmap_valgrind)((Addr) sb, cszB); 911b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov vg_assert2(! sr_isError(sres), "superblock valgrind munmap failure\n"); 912b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 913b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 914b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 915b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 916ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Find the superblock containing the given chunk. 917ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 918ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownSuperblock* findSb ( Arena* a, Block* b ) 919ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 920ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT min = 0; 921ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT max = a->sblocks_used; 922ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 923ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (min <= max) { 924ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Superblock * sb; 925ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT pos = min + (max - min)/2; 926ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 927ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(pos >= 0 && pos < a->sblocks_used); 928ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sb = a->sblocks[pos]; 929ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if ((Block*)&sb->payload_bytes[0] <= b 930ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown && b < (Block*)&sb->payload_bytes[sb->n_payload_bytes]) 931ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown { 932ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return sb; 933ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else if ((Block*)&sb->payload_bytes[0] <= b) { 934ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown min = pos + 1; 935ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 936ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown max = pos - 1; 937ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 938ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 939ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("findSb: can't find pointer %p in arena '%s'\n", 940ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b, a->name ); 941ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(core_panic)("findSb: VG_(arena_free)() in wrong arena?"); 942ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return NULL; /*NOTREACHED*/ 943ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 944ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 945ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 946eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov// Find the superblock containing the given address. 947eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov// If superblock not found, return NULL. 948eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanovstatic 949eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy IvanovSuperblock* maybe_findSb ( Arena* a, Addr ad ) 950eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov{ 951eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov SizeT min = 0; 952eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov SizeT max = a->sblocks_used; 953eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov 954eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov while (min <= max) { 955eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov Superblock * sb; 956eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov SizeT pos = min + (max - min)/2; 957eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov if (pos < 0 || pos >= a->sblocks_used) 958eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov return NULL; 959eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov sb = a->sblocks[pos]; 960eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov if ((Addr)&sb->payload_bytes[0] <= ad 961eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov && ad < (Addr)&sb->payload_bytes[sb->n_payload_bytes]) { 962eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov return sb; 963eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov } else if ((Addr)&sb->payload_bytes[0] <= ad) { 964eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov min = pos + 1; 965eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov } else { 966eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov max = pos - 1; 967eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov } 968eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov } 969eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov return NULL; 970eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov} 971eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov 972eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov 973ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------------------------------------------------------*/ 974ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--- Functions for working with freelists. ---*/ 975ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------------------------------------------------------*/ 976ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 977ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Nb: Determination of which freelist a block lives on is based on the 978ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// payload size, not block size. 979ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 980ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Convert a payload size in bytes to a freelist number. 981ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 982ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownUInt pszB_to_listNo ( SizeT pszB ) 983ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 984ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT n = pszB / VG_MIN_MALLOC_SZB; 985ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(0 == pszB % VG_MIN_MALLOC_SZB); 986ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 987ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // The first 64 lists hold blocks of size VG_MIN_MALLOC_SZB * list_num. 988ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // The final 48 hold bigger blocks. 989ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 64) return (UInt)n; 990ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Exponential slope up, factor 1.05 */ 991ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 67) return 64; 992ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 70) return 65; 993ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 74) return 66; 994ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 77) return 67; 995ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 81) return 68; 996ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 85) return 69; 997ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 90) return 70; 998ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 94) return 71; 999ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 99) return 72; 1000ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 104) return 73; 1001ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 109) return 74; 1002ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 114) return 75; 1003ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 120) return 76; 1004ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 126) return 77; 1005ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 133) return 78; 1006ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 139) return 79; 1007ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Exponential slope up, factor 1.10 */ 1008ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 153) return 80; 1009ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 169) return 81; 1010ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 185) return 82; 1011ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 204) return 83; 1012ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 224) return 84; 1013ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 247) return 85; 1014ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 272) return 86; 1015ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 299) return 87; 1016ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 329) return 88; 1017ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 362) return 89; 1018ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 398) return 90; 1019ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 438) return 91; 1020ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 482) return 92; 1021ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 530) return 93; 1022ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 583) return 94; 1023ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 641) return 95; 1024ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Exponential slope up, factor 1.20 */ 1025ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 770) return 96; 1026ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 924) return 97; 1027ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 1109) return 98; 1028ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 1331) return 99; 1029ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 1597) return 100; 1030ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 1916) return 101; 1031ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 2300) return 102; 1032ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 2760) return 103; 1033ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 3312) return 104; 1034ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 3974) return 105; 1035ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 4769) return 106; 1036ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 5723) return 107; 1037ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 6868) return 108; 1038ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 8241) return 109; 1039ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (n < 9890) return 110; 1040ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 111; 1041ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1042ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1043ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// What is the minimum payload size for a given list? 1044ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 1045ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownSizeT listNo_to_pszB_min ( UInt listNo ) 1046ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1047ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Repeatedly computing this function at every request is 1048ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown expensive. Hence at the first call just cache the result for 1049ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown every possible argument. */ 1050ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown static SizeT cache[N_MALLOC_LISTS]; 1051ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown static Bool cache_valid = False; 1052ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!cache_valid) { 1053ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt i; 1054ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_MALLOC_LISTS; i++) { 1055ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT pszB = 0; 1056ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (pszB_to_listNo(pszB) < i) 1057ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown pszB += VG_MIN_MALLOC_SZB; 1058ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cache[i] = pszB; 1059ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1060ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cache_valid = True; 1061ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1062ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Returned cached answer. */ 1063ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(listNo <= N_MALLOC_LISTS); 1064ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return cache[listNo]; 1065ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1066ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1067ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// What is the maximum payload size for a given list? 1068ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 1069ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownSizeT listNo_to_pszB_max ( UInt listNo ) 1070ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1071ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(listNo <= N_MALLOC_LISTS); 1072ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (listNo == N_MALLOC_LISTS-1) { 1073ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return MAX_PSZB; 1074ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 1075ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return listNo_to_pszB_min(listNo+1) - 1; 1076ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1077ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1078ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1079ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1080ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* A nasty hack to try and reduce fragmentation. Try and replace 1081ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a->freelist[lno] with another block on the same list but with a 1082ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lower address, with the idea of attempting to recycle the same 1083ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown blocks rather than cruise through the address space. */ 1084ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 1085ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid swizzle ( Arena* a, UInt lno ) 1086ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1087ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Block* p_best; 1088ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Block* pp; 1089ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Block* pn; 1090ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt i; 1091ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1092ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown p_best = a->freelist[lno]; 1093ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (p_best == NULL) return; 1094ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1095ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown pn = pp = p_best; 1096ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1097ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // This loop bound was 20 for a long time, but experiments showed that 1098ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // reducing it to 10 gave the same result in all the tests, and 5 got the 1099ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // same result in 85--100% of cases. And it's called often enough to be 1100ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // noticeable in programs that allocated a lot. 1101ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < 5; i++) { 1102ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown pn = get_next_b(pn); 1103ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown pp = get_prev_b(pp); 1104ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (pn < p_best) p_best = pn; 1105ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (pp < p_best) p_best = pp; 1106ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1107ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (p_best < a->freelist[lno]) { 1108ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# ifdef VERBOSE_MALLOC 1109ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("retreat by %ld\n", (Word)(a->freelist[lno] - p_best)); 1110ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# endif 1111ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a->freelist[lno] = p_best; 1112ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1113ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1114ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1115ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1116ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------------------------------------------------------*/ 1117ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--- Sanity-check/debugging machinery. ---*/ 1118ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------------------------------------------------------*/ 1119ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1120ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define REDZONE_LO_MASK 0x31 1121ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define REDZONE_HI_MASK 0x7c 1122ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1123ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Do some crude sanity checks on a Block. 1124ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 1125ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownBool blockSane ( Arena* a, Block* b ) 1126ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1127ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# define BLEAT(str) VG_(printf)("blockSane: fail -- %s\n",str) 1128ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt i; 1129ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // The lo and hi size fields will be checked (indirectly) by the call 1130ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // to get_rz_hi_byte(). 1131ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!a->clientmem && is_inuse_block(b)) { 1132663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // In the inner, for memcheck sake, temporarily mark redzone accessible. 1133663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng INNER_REQUEST(VALGRIND_MAKE_MEM_DEFINED 1134663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng (b + hp_overhead_szB() + sizeof(SizeT), a->rz_szB)); 1135663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng INNER_REQUEST(VALGRIND_MAKE_MEM_DEFINED 1136663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng (b + get_bszB(b) 1137663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng - sizeof(SizeT) - a->rz_szB, a->rz_szB)); 1138ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < a->rz_szB; i++) { 1139ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (get_rz_lo_byte(b, i) != 1140ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (UByte)(((Addr)b&0xff) ^ REDZONE_LO_MASK)) 1141ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown {BLEAT("redzone-lo");return False;} 1142ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (get_rz_hi_byte(b, i) != 1143ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (UByte)(((Addr)b&0xff) ^ REDZONE_HI_MASK)) 1144ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown {BLEAT("redzone-hi");return False;} 1145ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1146663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng INNER_REQUEST(VALGRIND_MAKE_MEM_NOACCESS 1147663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng (b + hp_overhead_szB() + sizeof(SizeT), a->rz_szB)); 1148663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng INNER_REQUEST(VALGRIND_MAKE_MEM_NOACCESS 1149663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng (b + get_bszB(b) 1150663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng - sizeof(SizeT) - a->rz_szB, a->rz_szB)); 1151ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1152ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return True; 1153ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# undef BLEAT 1154ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1155ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1156ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Print superblocks (only for debugging). 1157ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 1158ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid ppSuperblocks ( Arena* a ) 1159ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1160ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt i, j, blockno = 1; 1161ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT b_bszB; 1162ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1163ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (j = 0; j < a->sblocks_used; ++j) { 1164ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Superblock * sb = a->sblocks[j]; 1165ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1166ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( "\n" ); 1167b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(printf)( "superblock %d at %p %s, sb->n_pl_bs = %lu\n", 1168b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov blockno++, sb, (sb->unsplittable ? "unsplittable" : ""), 1169b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov sb->n_payload_bytes); 1170ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < sb->n_payload_bytes; i += b_bszB) { 1171ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Block* b = (Block*)&sb->payload_bytes[i]; 1172ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b_bszB = get_bszB(b); 1173ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( " block at %d, bszB %lu: ", i, b_bszB ); 1174ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( "%s, ", is_inuse_block(b) ? "inuse" : "free"); 1175ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( "%s\n", blockSane(a, b) ? "ok" : "BAD" ); 1176ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1177ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(i == sb->n_payload_bytes); // no overshoot at end of Sb 1178ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1179ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( "end of superblocks\n\n" ); 1180ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1181ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1182ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Sanity check both the superblocks and the chains. 1183ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void sanity_check_malloc_arena ( ArenaId aid ) 1184ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1185ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt i, j, superblockctr, blockctr_sb, blockctr_li; 1186ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt blockctr_sb_free, listno; 1187ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT b_bszB, b_pszB, list_min_pszB, list_max_pszB; 1188ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool thisFree, lastWasFree, sblockarrOK; 1189ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Block* b; 1190ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Block* b_prev; 1191ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT arena_bytes_on_loan; 1192ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Arena* a; 1193ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1194ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# define BOMB VG_(core_panic)("sanity_check_malloc_arena") 1195ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1196ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a = arenaId_to_ArenaP(aid); 1197ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1198ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // Check the superblock array. 1199ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sblockarrOK 1200ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown = a->sblocks != NULL 1201ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown && a->sblocks_size >= SBLOCKS_SIZE_INITIAL 1202ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown && a->sblocks_used <= a->sblocks_size 1203ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown && (a->sblocks_size == SBLOCKS_SIZE_INITIAL 1204ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ? (a->sblocks == &a->sblocks_initial[0]) 1205ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown : (a->sblocks != &a->sblocks_initial[0])); 1206ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!sblockarrOK) { 1207ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sanity_check_malloc_arena: sblock array BAD\n"); 1208ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown BOMB; 1209ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1210ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1211ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // First, traverse all the superblocks, inspecting the Blocks in each. 1212ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown superblockctr = blockctr_sb = blockctr_sb_free = 0; 1213ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown arena_bytes_on_loan = 0; 1214ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (j = 0; j < a->sblocks_used; ++j) { 1215ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Superblock * sb = a->sblocks[j]; 1216ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lastWasFree = False; 1217ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown superblockctr++; 1218ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < sb->n_payload_bytes; i += mk_plain_bszB(b_bszB)) { 1219ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown blockctr_sb++; 1220ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b = (Block*)&sb->payload_bytes[i]; 1221ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b_bszB = get_bszB_as_is(b); 1222ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!blockSane(a, b)) { 1223ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sanity_check_malloc_arena: sb %p, block %d " 1224ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "(bszB %lu): BAD\n", sb, i, b_bszB ); 1225ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown BOMB; 1226ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1227ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thisFree = !is_inuse_block(b); 1228ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (thisFree && lastWasFree) { 1229ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sanity_check_malloc_arena: sb %p, block %d " 1230ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "(bszB %lu): UNMERGED FREES\n", sb, i, b_bszB ); 1231ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown BOMB; 1232ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1233ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (thisFree) blockctr_sb_free++; 1234ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!thisFree) 1235ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown arena_bytes_on_loan += bszB_to_pszB(a, b_bszB); 1236ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lastWasFree = thisFree; 1237ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1238ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (i > sb->n_payload_bytes) { 1239ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( "sanity_check_malloc_arena: sb %p: last block " 1240ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "overshoots end\n", sb); 1241ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown BOMB; 1242ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1243ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1244ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1245436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov arena_bytes_on_loan += a->stats__perm_bytes_on_loan; 1246436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov 1247b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (arena_bytes_on_loan != a->stats__bytes_on_loan) { 1248ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# ifdef VERBOSE_MALLOC 1249b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(printf)( "sanity_check_malloc_arena: a->bytes_on_loan %lu, " 1250b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "arena_bytes_on_loan %lu: " 1251ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "MISMATCH\n", a->bytes_on_loan, arena_bytes_on_loan); 1252ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# endif 1253ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ppSuperblocks(a); 1254ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown BOMB; 1255ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1256ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1257ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Second, traverse each list, checking that the back pointers make 1258ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sense, counting blocks encountered, and checking that each block 1259ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown is an appropriate size for this list. */ 1260ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown blockctr_li = 0; 1261ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (listno = 0; listno < N_MALLOC_LISTS; listno++) { 1262ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown list_min_pszB = listNo_to_pszB_min(listno); 1263ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown list_max_pszB = listNo_to_pszB_max(listno); 1264ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b = a->freelist[listno]; 1265ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (b == NULL) continue; 1266ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (True) { 1267ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b_prev = b; 1268ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b = get_next_b(b); 1269ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (get_prev_b(b) != b_prev) { 1270ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( "sanity_check_malloc_arena: list %d at %p: " 1271ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "BAD LINKAGE\n", 1272ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown listno, b ); 1273ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown BOMB; 1274ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1275ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b_pszB = get_pszB(a, b); 1276ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (b_pszB < list_min_pszB || b_pszB > list_max_pszB) { 1277ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( 1278ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "sanity_check_malloc_arena: list %d at %p: " 1279ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "WRONG CHAIN SIZE %luB (%luB, %luB)\n", 1280ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown listno, b, b_pszB, list_min_pszB, list_max_pszB ); 1281ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown BOMB; 1282ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1283ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown blockctr_li++; 1284ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (b == a->freelist[listno]) break; 1285ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1286ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1287ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1288ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (blockctr_sb_free != blockctr_li) { 1289ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# ifdef VERBOSE_MALLOC 1290ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( "sanity_check_malloc_arena: BLOCK COUNT MISMATCH " 1291ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "(via sbs %d, via lists %d)\n", 1292ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown blockctr_sb_free, blockctr_li ); 1293ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# endif 1294ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ppSuperblocks(a); 1295ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown BOMB; 1296ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1297ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1298ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (VG_(clo_verbosity) > 2) 1299ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(message)(Vg_DebugMsg, 1300ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "%8s: %2d sbs, %5d bs, %2d/%-2d free bs, " 1301ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "%7ld mmap, %7ld loan\n", 1302ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a->name, 1303ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown superblockctr, 1304ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown blockctr_sb, blockctr_sb_free, blockctr_li, 1305b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->stats__bytes_mmaped, a->stats__bytes_on_loan); 1306ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# undef BOMB 1307ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1308ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1309ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1310ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#define N_AN_CCS 1000 1311ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1312436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovtypedef struct { 1313436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov ULong nBytes; 1314436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov ULong nBlocks; 1315436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov const HChar* cc; 1316436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov} AnCC; 1317ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1318ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic AnCC anCCs[N_AN_CCS]; 1319ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1320eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov/* Sorting by decreasing cost center nBytes, to have the biggest 1321eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov cost centres at the top. */ 1322436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovstatic Int cmp_AnCC_by_vol ( const void* v1, const void* v2 ) { 1323436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov const AnCC* ancc1 = v1; 1324436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov const AnCC* ancc2 = v2; 1325eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov if (ancc1->nBytes < ancc2->nBytes) return 1; 1326eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov if (ancc1->nBytes > ancc2->nBytes) return -1; 1327ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0; 1328ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1329ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1330ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic void cc_analyse_alloc_arena ( ArenaId aid ) 1331ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1332ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Word i, j, k; 1333ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Arena* a; 1334ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Block* b; 1335ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Bool thisFree, lastWasFree; 1336ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT b_bszB; 1337ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1338436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov const HChar* cc; 1339ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt n_ccs = 0; 1340ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown //return; 1341ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a = arenaId_to_ArenaP(aid); 1342ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (a->name == NULL) { 1343ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* arena is not in use, is not initialised and will fail the 1344ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sanity check that follows. */ 1345ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 1346ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1347ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1348ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sanity_check_malloc_arena(aid); 1349ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1350ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( 1351b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "-------- Arena \"%s\": %lu/%lu max/curr mmap'd, " 1352b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "%llu/%llu unsplit/split sb unmmap'd, " 1353663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng "%lu/%lu max/curr on_loan %lu rzB --------\n", 1354b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->name, a->stats__bytes_mmaped_max, a->stats__bytes_mmaped, 1355b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->stats__nreclaim_unsplit, a->stats__nreclaim_split, 1356663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng a->stats__bytes_on_loan_max, a->stats__bytes_on_loan, 1357663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng a->rz_szB 1358ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ); 1359ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1360ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (j = 0; j < a->sblocks_used; ++j) { 1361ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Superblock * sb = a->sblocks[j]; 1362ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lastWasFree = False; 1363ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < sb->n_payload_bytes; i += mk_plain_bszB(b_bszB)) { 1364ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b = (Block*)&sb->payload_bytes[i]; 1365ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b_bszB = get_bszB_as_is(b); 1366ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!blockSane(a, b)) { 1367ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sanity_check_malloc_arena: sb %p, block %ld " 1368ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "(bszB %lu): BAD\n", sb, i, b_bszB ); 1369ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 1370ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1371ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown thisFree = !is_inuse_block(b); 1372ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (thisFree && lastWasFree) { 1373ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("sanity_check_malloc_arena: sb %p, block %ld " 1374ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "(bszB %lu): UNMERGED FREES\n", sb, i, b_bszB ); 1375ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 1376ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1377ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lastWasFree = thisFree; 1378ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1379ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (thisFree) continue; 1380ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1381ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0) 1382ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("block: inUse=%d pszB=%d cc=%s\n", 1383ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)(!thisFree), 1384ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (Int)bszB_to_pszB(a, b_bszB), 1385ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown get_cc(b)); 1386ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown cc = get_cc(b); 1387ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(cc); 1388ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (k = 0; k < n_ccs; k++) { 1389ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(anCCs[k].cc); 1390ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0 == VG_(strcmp)(cc, anCCs[k].cc)) 1391ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1392ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1393ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(k >= 0 && k <= n_ccs); 1394ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1395ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (k == n_ccs) { 1396ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(n_ccs < N_AN_CCS-1); 1397ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown n_ccs++; 1398ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown anCCs[k].nBytes = 0; 1399ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown anCCs[k].nBlocks = 0; 1400ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown anCCs[k].cc = cc; 1401ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1402ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1403ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(k >= 0 && k < n_ccs && k < N_AN_CCS); 1404ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown anCCs[k].nBytes += (ULong)bszB_to_pszB(a, b_bszB); 1405ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown anCCs[k].nBlocks++; 1406ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1407ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (i > sb->n_payload_bytes) { 1408ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)( "sanity_check_malloc_arena: sb %p: last block " 1409ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "overshoots end\n", sb); 1410ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown tl_assert(0); 1411ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1412ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1413ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1414436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov if (a->stats__perm_bytes_on_loan > 0) { 1415436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov tl_assert(n_ccs < N_AN_CCS-1); 1416436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov anCCs[n_ccs].nBytes = a->stats__perm_bytes_on_loan; 1417436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov anCCs[n_ccs].nBlocks = a->stats__perm_blocks; 1418436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov anCCs[n_ccs].cc = "perm_malloc"; 1419436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov n_ccs++; 1420436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov } 1421436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov 1422ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(ssort)( &anCCs[0], n_ccs, sizeof(anCCs[0]), cmp_AnCC_by_vol ); 1423ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1424ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (k = 0; k < n_ccs; k++) { 1425ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("%'13llu in %'9llu: %s\n", 1426ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown anCCs[k].nBytes, anCCs[k].nBlocks, anCCs[k].cc ); 1427ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1428ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1429ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("\n"); 1430ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1431ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1432ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1433ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid VG_(sanity_check_malloc_all) ( void ) 1434ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1435ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt i; 1436ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < VG_N_ARENAS; i++) { 1437ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (i == VG_AR_CLIENT && !client_inited) 1438ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown continue; 1439ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sanity_check_malloc_arena ( i ); 1440ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1441ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1442ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1443eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanovvoid VG_(describe_arena_addr) ( Addr a, AddrArenaInfo* aai ) 1444eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov{ 1445eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov UInt i; 1446eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov Superblock *sb; 1447eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov Arena *arena; 1448eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov 1449eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov for (i = 0; i < VG_N_ARENAS; i++) { 1450eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov if (i == VG_AR_CLIENT && !client_inited) 1451eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov continue; 1452eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov arena = arenaId_to_ArenaP(i); 1453eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov sb = maybe_findSb( arena, a ); 1454eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov if (sb != NULL) { 1455eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov Word j; 1456eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov SizeT b_bszB; 1457eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov Block *b = NULL; 1458eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov 1459eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov aai->aid = i; 1460eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov aai->name = arena->name; 1461eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov for (j = 0; j < sb->n_payload_bytes; j += mk_plain_bszB(b_bszB)) { 1462eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov b = (Block*)&sb->payload_bytes[j]; 1463eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov b_bszB = get_bszB_as_is(b); 1464eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov if (a < (Addr)b + mk_plain_bszB(b_bszB)) 1465eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov break; 1466eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov } 1467eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov vg_assert (b); 1468eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov aai->block_szB = get_pszB(arena, b); 1469eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov aai->rwoffset = a - (Addr)get_block_payload(arena, b); 1470eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov aai->free = !is_inuse_block(b); 1471eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov return; 1472eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov } 1473eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov } 1474eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov aai->aid = 0; 1475eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov aai->name = NULL; 1476eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov aai->block_szB = 0; 1477eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov aai->rwoffset = 0; 1478eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov aai->free = False; 1479eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov} 1480ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1481ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------------------------------------------------------*/ 1482ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--- Creating and deleting blocks. ---*/ 1483ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------------------------------------------------------*/ 1484ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1485ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Mark the bytes at b .. b+bszB-1 as not in use, and add them to the 1486ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// relevant free list. 1487ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1488ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 1489ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid mkFreeBlock ( Arena* a, Block* b, SizeT bszB, UInt b_lno ) 1490ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1491ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT pszB = bszB_to_pszB(a, bszB); 1492ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(b_lno == pszB_to_listNo(pszB)); 1493663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng INNER_REQUEST(VALGRIND_MAKE_MEM_UNDEFINED(b, bszB)); 1494ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // Set the size fields and indicate not-in-use. 1495ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown set_bszB(b, mk_free_bszB(bszB)); 1496ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1497ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // Add to the relevant list. 1498ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (a->freelist[b_lno] == NULL) { 1499ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown set_prev_b(b, b); 1500ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown set_next_b(b, b); 1501ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a->freelist[b_lno] = b; 1502ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 1503ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Block* b_prev = get_prev_b(a->freelist[b_lno]); 1504ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Block* b_next = a->freelist[b_lno]; 1505ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown set_next_b(b_prev, b); 1506ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown set_prev_b(b_next, b); 1507ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown set_next_b(b, b_next); 1508ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown set_prev_b(b, b_prev); 1509ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1510ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# ifdef DEBUG_MALLOC 1511ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (void)blockSane(a,b); 1512ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# endif 1513ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1514ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1515ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Mark the bytes at b .. b+bszB-1 as in use, and set up the block 1516ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// appropriately. 1517ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 1518ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid mkInuseBlock ( Arena* a, Block* b, SizeT bszB ) 1519ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1520ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt i; 1521ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(bszB >= min_useful_bszB(a)); 1522663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng INNER_REQUEST(VALGRIND_MAKE_MEM_UNDEFINED(b, bszB)); 1523ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown set_bszB(b, mk_inuse_bszB(bszB)); 1524ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown set_prev_b(b, NULL); // Take off freelist 1525ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown set_next_b(b, NULL); // ditto 1526ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!a->clientmem) { 1527ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < a->rz_szB; i++) { 1528ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown set_rz_lo_byte(b, i, (UByte)(((Addr)b&0xff) ^ REDZONE_LO_MASK)); 1529ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown set_rz_hi_byte(b, i, (UByte)(((Addr)b&0xff) ^ REDZONE_HI_MASK)); 1530ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1531ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1532ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# ifdef DEBUG_MALLOC 1533ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (void)blockSane(a,b); 1534ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# endif 1535ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1536ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1537ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Remove a block from a given list. Does no sanity checking. 1538ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic 1539ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid unlinkBlock ( Arena* a, Block* b, UInt listno ) 1540ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1541ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(listno < N_MALLOC_LISTS); 1542ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (get_prev_b(b) == b) { 1543ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // Only one element in the list; treat it specially. 1544ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(get_next_b(b) == b); 1545ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a->freelist[listno] = NULL; 1546ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 1547ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Block* b_prev = get_prev_b(b); 1548ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Block* b_next = get_next_b(b); 1549ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a->freelist[listno] = b_prev; 1550ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown set_next_b(b_prev, b_next); 1551ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown set_prev_b(b_next, b_prev); 1552ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown swizzle ( a, listno ); 1553ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1554ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown set_prev_b(b, NULL); 1555ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown set_next_b(b, NULL); 1556ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1557ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1558ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1559ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------------------------------------------------------*/ 1560ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--- Core-visible functions. ---*/ 1561ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------------------------------------------------------*/ 1562ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1563ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Align the request size. 1564ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownstatic __inline__ 1565ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownSizeT align_req_pszB ( SizeT req_pszB ) 1566ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1567ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT n = VG_MIN_MALLOC_SZB-1; 1568ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ((req_pszB + n) & (~n)); 1569ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1570ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1571436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovstatic 1572436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovvoid add_one_block_to_stats (Arena* a, SizeT loaned) 1573436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov{ 1574436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov a->stats__bytes_on_loan += loaned; 1575436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov if (a->stats__bytes_on_loan > a->stats__bytes_on_loan_max) { 1576436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov a->stats__bytes_on_loan_max = a->stats__bytes_on_loan; 1577436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov if (a->stats__bytes_on_loan_max >= a->next_profile_at) { 1578436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov /* next profile after 10% more growth */ 1579436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov a->next_profile_at 1580436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov = (SizeT)( 1581436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov (((ULong)a->stats__bytes_on_loan_max) * 105ULL) / 100ULL ); 1582436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov if (VG_(clo_profile_heap)) 1583436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov cc_analyse_alloc_arena(arenaP_to_ArenaId (a)); 1584436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov } 1585436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov } 1586436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov a->stats__tot_blocks += (ULong)1; 1587436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov a->stats__tot_bytes += (ULong)loaned; 1588436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov} 1589436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov 1590436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovvoid* VG_(arena_malloc) ( ArenaId aid, const HChar* cc, SizeT req_pszB ) 1591ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1592ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT req_bszB, frag_bszB, b_bszB; 1593ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt lno, i; 1594b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Superblock* new_sb = NULL; 1595ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Block* b = NULL; 1596ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Arena* a; 1597ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown void* v; 1598b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UWord stats__nsearches = 0; 1599ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1600ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ensure_mm_init(aid); 1601ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a = arenaId_to_ArenaP(aid); 1602ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1603ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(req_pszB < MAX_PSZB); 1604ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown req_pszB = align_req_pszB(req_pszB); 1605ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown req_bszB = pszB_to_bszB(a, req_pszB); 1606ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1607ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // You must provide a cost-center name against which to charge 1608ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // this allocation; it isn't optional. 1609ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(cc); 1610ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1611ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // Scan through all the big-enough freelists for a block. 1612ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // 1613ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // Nb: this scanning might be expensive in some cases. Eg. if you 1614ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // allocate lots of small objects without freeing them, but no 1615ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // medium-sized objects, it will repeatedly scanning through the whole 1616ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // list, and each time not find any free blocks until the last element. 1617ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // 1618ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // If this becomes a noticeable problem... the loop answers the question 1619ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // "where is the first nonempty list above me?" And most of the time, 1620ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // you ask the same question and get the same answer. So it would be 1621ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // good to somehow cache the results of previous searches. 1622ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // One possibility is an array (with N_MALLOC_LISTS elements) of 1623ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // shortcuts. shortcut[i] would give the index number of the nearest 1624ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // larger list above list i which is non-empty. Then this loop isn't 1625ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // necessary. However, we'd have to modify some section [ .. i-1] of the 1626ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // shortcut array every time a list [i] changes from empty to nonempty or 1627ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // back. This would require care to avoid pathological worst-case 1628ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // behaviour. 1629ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // 1630ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (lno = pszB_to_listNo(req_pszB); lno < N_MALLOC_LISTS; lno++) { 1631b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UWord nsearches_this_level = 0; 1632ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b = a->freelist[lno]; 1633ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (NULL == b) continue; // If this list is empty, try the next one. 1634ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (True) { 1635b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov stats__nsearches++; 1636b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov nsearches_this_level++; 1637b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (UNLIKELY(nsearches_this_level >= 100) 1638b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov && lno < N_MALLOC_LISTS-1) { 1639b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* Avoid excessive scanning on this freelist, and instead 1640b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov try the next one up. But first, move this freelist's 1641b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov start pointer one element along, so as to ensure that 1642b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov subsequent searches of this list don't endlessly 1643b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov revisit only these 100 elements, but in fact slowly 1644b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov progress through the entire list. */ 1645b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov b = a->freelist[lno]; 1646b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov vg_assert(b); // this list must be nonempty! 1647b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->freelist[lno] = get_next_b(b); // step one along 1648b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov break; 1649b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 1650ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b_bszB = get_bszB(b); 1651ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (b_bszB >= req_bszB) goto obtained_block; // success! 1652ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b = get_next_b(b); 1653ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (b == a->freelist[lno]) break; // traversed entire freelist 1654ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1655ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1656ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1657ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // If we reach here, no suitable block found, allocate a new superblock 1658ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(lno == N_MALLOC_LISTS); 1659ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown new_sb = newSuperblock(a, req_bszB); 1660ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (NULL == new_sb) { 1661ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // Should only fail if for client, otherwise, should have aborted 1662ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // already. 1663ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(VG_AR_CLIENT == aid); 1664ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return NULL; 1665ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1666ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1667ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(a->sblocks_used <= a->sblocks_size); 1668ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (a->sblocks_used == a->sblocks_size) { 1669ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Superblock ** array; 1670eb0bae136f4eeaaf29761dddb148b118fb824632Dmitriy Ivanov SysRes sres = VG_(am_mmap_anon_float_valgrind)(sizeof(Superblock *) * 1671ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a->sblocks_size * 2); 1672ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (sr_isError(sres)) { 1673ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(out_of_memory_NORETURN)("arena_init", sizeof(Superblock *) * 1674ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a->sblocks_size * 2); 1675ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* NOTREACHED */ 1676ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1677ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown array = (Superblock**)(AddrH)sr_Res(sres); 1678ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < a->sblocks_used; ++i) array[i] = a->sblocks[i]; 1679ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1680ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a->sblocks_size *= 2; 1681ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a->sblocks = array; 1682ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(debugLog)(1, "mallocfree", 1683ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "sblock array for arena `%s' resized to %ld\n", 1684ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a->name, a->sblocks_size); 1685ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1686ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1687ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(a->sblocks_used < a->sblocks_size); 1688ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1689ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown i = a->sblocks_used; 1690ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (i > 0) { 1691ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (a->sblocks[i-1] > new_sb) { 1692ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a->sblocks[i] = a->sblocks[i-1]; 1693ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 1694ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown break; 1695ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1696ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown --i; 1697ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1698ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a->sblocks[i] = new_sb; 1699ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a->sblocks_used++; 1700ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1701ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b = (Block*)&new_sb->payload_bytes[0]; 1702ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown lno = pszB_to_listNo(bszB_to_pszB(a, new_sb->n_payload_bytes)); 1703ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mkFreeBlock ( a, b, new_sb->n_payload_bytes, lno); 1704ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (VG_(clo_profile_heap)) 1705ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown set_cc(b, "admin.free-new-sb-1"); 1706ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // fall through 1707ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1708ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown obtained_block: 1709ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // Ok, we can allocate from b, which lives in list lno. 1710ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(b != NULL); 1711ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(lno < N_MALLOC_LISTS); 1712ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(a->freelist[lno] != NULL); 1713ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b_bszB = get_bszB(b); 1714ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // req_bszB is the size of the block we are after. b_bszB is the 1715ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // size of what we've actually got. */ 1716ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(b_bszB >= req_bszB); 1717ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1718ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // Could we split this block and still get a useful fragment? 1719b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // A block in an unsplittable superblock can never be splitted. 1720ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown frag_bszB = b_bszB - req_bszB; 1721b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (frag_bszB >= min_useful_bszB(a) 1722b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov && (NULL == new_sb || ! new_sb->unsplittable)) { 1723ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // Yes, split block in two, put the fragment on the appropriate free 1724ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // list, and update b_bszB accordingly. 1725ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // printf( "split %dB into %dB and %dB\n", b_bszB, req_bszB, frag_bszB ); 1726ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown unlinkBlock(a, b, lno); 1727ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mkInuseBlock(a, b, req_bszB); 1728ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (VG_(clo_profile_heap)) 1729ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown set_cc(b, cc); 1730ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mkFreeBlock(a, &b[req_bszB], frag_bszB, 1731ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown pszB_to_listNo(bszB_to_pszB(a, frag_bszB))); 1732ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (VG_(clo_profile_heap)) 1733ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown set_cc(&b[req_bszB], "admin.fragmentation-1"); 1734ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b_bszB = get_bszB(b); 1735ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 1736ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // No, mark as in use and use as-is. 1737ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown unlinkBlock(a, b, lno); 1738ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mkInuseBlock(a, b, b_bszB); 1739ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (VG_(clo_profile_heap)) 1740ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown set_cc(b, cc); 1741ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1742ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1743ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // Update stats 1744b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov SizeT loaned = bszB_to_pszB(a, b_bszB); 1745436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov add_one_block_to_stats (a, loaned); 1746b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->stats__nsearches += (ULong)stats__nsearches; 1747ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1748ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# ifdef DEBUG_MALLOC 1749ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sanity_check_malloc_arena(aid); 1750ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# endif 1751ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1752ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown v = get_block_payload(a, b); 1753ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert( (((Addr)v) & (VG_MIN_MALLOC_SZB-1)) == 0 ); 1754ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1755663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // Which size should we pass to VALGRIND_MALLOCLIKE_BLOCK ? 1756663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // We have 2 possible options: 1757663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // 1. The final resulting usable size. 1758663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // 2. The initial (non-aligned) req_pszB. 1759663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // Memcheck implements option 2 easily, as the initial requested size 1760663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // is maintained in the mc_chunk data structure. 1761663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // This is not as easy in the core, as there is no such structure. 1762663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // (note: using the aligned req_pszB is not simpler than 2, as 1763663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // requesting an aligned req_pszB might still be satisfied by returning 1764663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // a (slightly) bigger block than requested if the remaining part of 1765663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // of a free block is not big enough to make a free block by itself). 1766663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // Implement Sol 2 can be done the following way: 1767663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // After having called VALGRIND_MALLOCLIKE_BLOCK, the non accessible 1768663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // redzone just after the block can be used to determine the 1769663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // initial requested size. 1770663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // Currently, not implemented => we use Option 1. 1771663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng INNER_REQUEST 1772663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng (VALGRIND_MALLOCLIKE_BLOCK(v, 1773663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng VG_(arena_malloc_usable_size)(aid, v), 1774663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng a->rz_szB, False)); 1775ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1776ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* For debugging/testing purposes, fill the newly allocated area 1777ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown with a definite value in an attempt to shake out any 1778ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown uninitialised uses of the data (by V core / V tools, not by the 1779ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown client). Testing on 25 Nov 07 with the values 0x00, 0xFF, 0x55, 1780ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 0xAA showed no differences in the regression tests on 1781ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown amd64-linux. Note, is disabled by default. */ 1782ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0 && aid != VG_AR_CLIENT) 1783ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(memset)(v, 0xAA, (SizeT)req_pszB); 1784ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1785ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return v; 1786ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 1787ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1788b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// If arena has already a deferred reclaimed superblock and 1789b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// this superblock is still reclaimable, then this superblock is first 1790b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// reclaimed. 1791b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// sb becomes then the new arena deferred superblock. 1792b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// Passing NULL as sb allows to reclaim a deferred sb without setting a new 1793b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov// deferred reclaim. 1794b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovstatic 1795b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovvoid deferred_reclaimSuperblock ( Arena* a, Superblock* sb) 1796b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 1797b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1798b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (sb == NULL) { 1799b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (!a->deferred_reclaimed_sb) 1800b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // no deferred sb to reclaim now, nothing to do in the future => 1801b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // return directly. 1802b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov return; 1803b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1804b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(debugLog)(1, "mallocfree", 1805b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "deferred_reclaimSuperblock NULL " 1806b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "(prev %p) owner %s/%s\n", 1807b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->deferred_reclaimed_sb, 1808b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->clientmem ? "CLIENT" : "VALGRIND", a->name ); 1809b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } else 1810b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov VG_(debugLog)(1, "mallocfree", 1811b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "deferred_reclaimSuperblock at %p (pszB %7ld) %s " 1812b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "(prev %p) owner %s/%s\n", 1813b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov sb, sb->n_payload_bytes, 1814b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov (sb->unsplittable ? "unsplittable" : ""), 1815b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->deferred_reclaimed_sb, 1816b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->clientmem ? "CLIENT" : "VALGRIND", a->name ); 1817b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1818b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (a->deferred_reclaimed_sb && a->deferred_reclaimed_sb != sb) { 1819b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // If we are deferring another block that the current block deferred, 1820b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // then if this block can stil be reclaimed, reclaim it now. 1821b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // Note that we might have a re-deferred reclaim of the same block 1822b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // with a sequence: free (causing a deferred reclaim of sb) 1823b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // alloc (using a piece of memory of the deferred sb) 1824b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // free of the just alloc-ed block (causing a re-defer). 1825b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UByte* def_sb_start; 1826b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UByte* def_sb_end; 1827b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Superblock* def_sb; 1828b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov Block* b; 1829b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1830b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov def_sb = a->deferred_reclaimed_sb; 1831b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov def_sb_start = &def_sb->payload_bytes[0]; 1832b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov def_sb_end = &def_sb->payload_bytes[def_sb->n_payload_bytes - 1]; 1833b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov b = (Block *)def_sb_start; 1834b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov vg_assert (blockSane(a, b)); 1835b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1836b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // Check if the deferred_reclaimed_sb is still reclaimable. 1837b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // If yes, we will execute the reclaim. 1838b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (!is_inuse_block(b)) { 1839b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // b (at the beginning of def_sb) is not in use. 1840b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov UInt b_listno; 1841b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov SizeT b_bszB, b_pszB; 1842b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov b_bszB = get_bszB(b); 1843b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov b_pszB = bszB_to_pszB(a, b_bszB); 1844b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (b + b_bszB-1 == (Block*)def_sb_end) { 1845b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // b (not in use) covers the full superblock. 1846b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // => def_sb is still reclaimable 1847b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // => execute now the reclaim of this def_sb. 1848b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov b_listno = pszB_to_listNo(b_pszB); 1849b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov unlinkBlock( a, b, b_listno ); 1850b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov reclaimSuperblock (a, def_sb); 1851b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->deferred_reclaimed_sb = NULL; 1852b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 1853b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 1854b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 1855b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1856b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // sb (possibly NULL) becomes the new deferred reclaimed superblock. 1857b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->deferred_reclaimed_sb = sb; 1858b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 1859b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1860ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1861ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid VG_(arena_free) ( ArenaId aid, void* ptr ) 1862ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 1863ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Superblock* sb; 1864ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UByte* sb_start; 1865ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UByte* sb_end; 1866ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Block* other_b; 1867ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Block* b; 1868ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT b_bszB, b_pszB, other_bszB; 1869ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UInt b_listno; 1870ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Arena* a; 1871ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1872ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ensure_mm_init(aid); 1873ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a = arenaId_to_ArenaP(aid); 1874ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1875ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (ptr == NULL) { 1876ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 1877ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 1878ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1879ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b = get_payload_block(a, ptr); 1880ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1881ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* If this is one of V's areas, check carefully the block we're 1882ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown getting back. This picks up simple block-end overruns. */ 1883ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (aid != VG_AR_CLIENT) 1884ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(blockSane(a, b)); 1885ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1886ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b_bszB = get_bszB(b); 1887ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b_pszB = bszB_to_pszB(a, b_bszB); 1888ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sb = findSb( a, b ); 1889ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sb_start = &sb->payload_bytes[0]; 1890ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sb_end = &sb->payload_bytes[sb->n_payload_bytes - 1]; 1891ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1892b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->stats__bytes_on_loan -= b_pszB; 1893ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1894ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* If this is one of V's areas, fill it up with junk to enhance the 1895ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown chances of catching any later reads of it. Note, 0xDD is 1896ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown carefully chosen junk :-), in that: (1) 0xDDDDDDDD is an invalid 1897ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown and non-word-aligned address on most systems, and (2) 0xDD is a 1898ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown value which is unlikely to be generated by the new compressed 1899ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Vbits representation for memcheck. */ 1900ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (aid != VG_AR_CLIENT) 1901ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(memset)(ptr, 0xDD, (SizeT)b_pszB); 1902ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 1903b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (! sb->unsplittable) { 1904b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // Put this chunk back on a list somewhere. 1905b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov b_listno = pszB_to_listNo(b_pszB); 1906b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov mkFreeBlock( a, b, b_bszB, b_listno ); 1907b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (VG_(clo_profile_heap)) 1908b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov set_cc(b, "admin.free-1"); 1909b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1910b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // See if this block can be merged with its successor. 1911b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // First test if we're far enough before the superblock's end to possibly 1912b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // have a successor. 1913b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov other_b = b + b_bszB; 1914b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (other_b+min_useful_bszB(a)-1 <= (Block*)sb_end) { 1915b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // Ok, we have a successor, merge if it's not in use. 1916b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov other_bszB = get_bszB(other_b); 1917b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (!is_inuse_block(other_b)) { 1918b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // VG_(printf)( "merge-successor\n"); 1919b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov# ifdef DEBUG_MALLOC 1920b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov vg_assert(blockSane(a, other_b)); 1921b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov# endif 1922b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov unlinkBlock( a, b, b_listno ); 1923b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov unlinkBlock( a, other_b, 1924b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov pszB_to_listNo(bszB_to_pszB(a,other_bszB)) ); 1925b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov b_bszB += other_bszB; 1926b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov b_listno = pszB_to_listNo(bszB_to_pszB(a, b_bszB)); 1927b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov mkFreeBlock( a, b, b_bszB, b_listno ); 1928b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (VG_(clo_profile_heap)) 1929b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov set_cc(b, "admin.free-2"); 1930b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 1931b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } else { 1932b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // Not enough space for successor: check that b is the last block 1933b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // ie. there are no unused bytes at the end of the Superblock. 1934b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov vg_assert(other_b-1 == (Block*)sb_end); 1935ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 19368f943afc22a6a683b78271836c8ddc462b4824a9Evgeniy Stepanov 1937b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // Then see if this block can be merged with its predecessor. 1938b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // First test if we're far enough after the superblock's start to possibly 1939b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // have a predecessor. 1940b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (b >= (Block*)sb_start + min_useful_bszB(a)) { 1941b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // Ok, we have a predecessor, merge if it's not in use. 1942b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov other_b = get_predecessor_block( b ); 1943b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov other_bszB = get_bszB(other_b); 1944b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (!is_inuse_block(other_b)) { 1945b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // VG_(printf)( "merge-predecessor\n"); 1946b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov unlinkBlock( a, b, b_listno ); 1947b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov unlinkBlock( a, other_b, 1948b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov pszB_to_listNo(bszB_to_pszB(a, other_bszB)) ); 1949b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov b = other_b; 1950b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov b_bszB += other_bszB; 1951b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov b_listno = pszB_to_listNo(bszB_to_pszB(a, b_bszB)); 1952b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov mkFreeBlock( a, b, b_bszB, b_listno ); 1953b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (VG_(clo_profile_heap)) 1954b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov set_cc(b, "admin.free-3"); 1955b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 1956b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } else { 1957b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // Not enough space for predecessor: check that b is the first block, 1958b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // ie. there are no unused bytes at the start of the Superblock. 1959b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov vg_assert((Block*)sb_start == b); 19608f943afc22a6a683b78271836c8ddc462b4824a9Evgeniy Stepanov } 1961b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1962b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* If the block b just merged is the only block of the superblock sb, 1963b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov then we defer reclaim sb. */ 1964b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if ( ((Block*)sb_start == b) && (b + b_bszB-1 == (Block*)sb_end) ) { 1965b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov deferred_reclaimSuperblock (a, sb); 1966b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 1967b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1968663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // Inform that ptr has been released. We give redzone size 1969663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // 0 instead of a->rz_szB as proper accessibility is done just after. 1970663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng INNER_REQUEST(VALGRIND_FREELIKE_BLOCK(ptr, 0)); 1971663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng 1972663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // We need to (re-)establish the minimum accessibility needed 1973663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // for free list management. E.g. if block ptr has been put in a free 1974663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // list and a neighbour block is released afterwards, the 1975663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // "lo" and "hi" portions of the block ptr will be accessed to 1976663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // glue the 2 blocks together. 1977663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // We could mark the whole block as not accessible, and each time 1978663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // transiently mark accessible the needed lo/hi parts. Not done as this 1979663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // is quite complex, for very little expected additional bug detection. 1980663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // fully unaccessible. Note that the below marks the (possibly) merged 1981663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // block, not the block corresponding to the ptr argument. 1982663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng 1983663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // First mark the whole block unaccessible. 1984663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng INNER_REQUEST(VALGRIND_MAKE_MEM_NOACCESS(b, b_bszB)); 1985663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // Then mark the relevant administrative headers as defined. 1986663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // No need to mark the heap profile portion as defined, this is not 1987663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // used for free blocks. 1988663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng INNER_REQUEST(VALGRIND_MAKE_MEM_DEFINED(b + hp_overhead_szB(), 1989663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng sizeof(SizeT) + sizeof(void*))); 1990663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng INNER_REQUEST(VALGRIND_MAKE_MEM_DEFINED(b + b_bszB 1991663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng - sizeof(SizeT) - sizeof(void*), 1992663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng sizeof(SizeT) + sizeof(void*))); 1993ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 1994b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // b must be first block (i.e. no unused bytes at the beginning) 1995ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert((Block*)sb_start == b); 1996b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 1997b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // b must be last block (i.e. no unused bytes at the end) 1998b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov other_b = b + b_bszB; 1999b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov vg_assert(other_b-1 == (Block*)sb_end); 2000b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 2001663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // Inform that ptr has been released. Redzone size value 2002663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // is not relevant (so we give 0 instead of a->rz_szB) 2003663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // as it is expected that the aspacemgr munmap will be used by 2004663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // outer to mark the whole superblock as unaccessible. 2005663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng INNER_REQUEST(VALGRIND_FREELIKE_BLOCK(ptr, 0)); 2006663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng 2007b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov // Reclaim immediately the unsplittable superblock sb. 2008b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov reclaimSuperblock (a, sb); 2009ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2010ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2011ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# ifdef DEBUG_MALLOC 2012ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sanity_check_malloc_arena(aid); 2013ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# endif 2014ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2015ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2016ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2017ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2018ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* 2019ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown The idea for malloc_aligned() is to allocate a big block, base, and 2020ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown then split it into two parts: frag, which is returned to the the 2021ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown free pool, and align, which is the bit we're really after. Here's 2022ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a picture. L and H denote the block lower and upper overheads, in 2023ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown bytes. The details are gruesome. Note it is slightly complicated 2024ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown because the initial request to generate base may return a bigger 2025ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown block than we asked for, so it is important to distinguish the base 2026ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown request size and the base actual size. 2027ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2028ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown frag_b align_b 2029ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | | 2030ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | frag_p | align_p 2031ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | | | | 2032ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown v v v v 2033ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2034ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown +---+ +---+---+ +---+ 2035ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | L |----------------| H | L |---------------| H | 2036ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown +---+ +---+---+ +---+ 2037ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2038ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ^ ^ ^ 2039ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | | : 2040ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | base_p this addr must be aligned 2041ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown | 2042ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown base_b 2043ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2044ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown . . . . . . . 2045ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown <------ frag_bszB -------> . . . 2046ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown . <------------- base_pszB_act -----------> . 2047ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown . . . . . . . 2048ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2049ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown*/ 2050436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovvoid* VG_(arena_memalign) ( ArenaId aid, const HChar* cc, 2051ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT req_alignB, SizeT req_pszB ) 2052ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2053ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT base_pszB_req, base_pszB_act, frag_bszB; 2054ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Block *base_b, *align_b; 2055ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UByte *base_p, *align_p; 2056ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT saved_bytes_on_loan; 2057ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Arena* a; 2058ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2059ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ensure_mm_init(aid); 2060ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a = arenaId_to_ArenaP(aid); 2061ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2062ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(req_pszB < MAX_PSZB); 2063ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2064ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // You must provide a cost-center name against which to charge 2065ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // this allocation; it isn't optional. 2066ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(cc); 2067ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2068663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng // Check that the requested alignment has a plausible size. 2069ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // Check that the requested alignment seems reasonable; that is, is 2070ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // a power of 2. 2071ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (req_alignB < VG_MIN_MALLOC_SZB 2072663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng || req_alignB > 16 * 1024 * 1024 2073ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown || VG_(log2)( req_alignB ) == -1 /* not a power of 2 */) { 2074ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(printf)("VG_(arena_memalign)(%p, %lu, %lu)\n" 2075ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "bad alignment value %lu\n" 2076ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "(it is too small, too big, or not a power of two)", 2077ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a, req_alignB, req_pszB, req_alignB ); 2078ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(core_panic)("VG_(arena_memalign)"); 2079ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /*NOTREACHED*/ 2080ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2081ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // Paranoid 2082ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(req_alignB % VG_MIN_MALLOC_SZB == 0); 2083ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2084ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Required payload size for the aligned chunk. */ 2085ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown req_pszB = align_req_pszB(req_pszB); 2086ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2087ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Payload size to request for the big block that we will split up. */ 2088ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown base_pszB_req = req_pszB + min_useful_bszB(a) + req_alignB; 2089ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2090ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Payload ptr for the block we are going to split. Note this 2091ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown changes a->bytes_on_loan; we save and restore it ourselves. */ 2092b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov saved_bytes_on_loan = a->stats__bytes_on_loan; 2093b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov { 2094b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* As we will split the block given back by VG_(arena_malloc), 2095b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov we have to (temporarily) disable unsplittable for this arena, 2096b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov as unsplittable superblocks cannot be splitted. */ 2097b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov const SizeT save_min_unsplittable_sblock_szB 2098b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov = a->min_unsplittable_sblock_szB; 2099b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->min_unsplittable_sblock_szB = MAX_PSZB; 2100b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov base_p = VG_(arena_malloc) ( aid, cc, base_pszB_req ); 2101b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->min_unsplittable_sblock_szB = save_min_unsplittable_sblock_szB; 2102b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 2103b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->stats__bytes_on_loan = saved_bytes_on_loan; 2104ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2105ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Give up if we couldn't allocate enough space */ 2106ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (base_p == 0) 2107ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return 0; 2108663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng /* base_p was marked as allocated by VALGRIND_MALLOCLIKE_BLOCK 2109663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng inside VG_(arena_malloc). We need to indicate it is free, then 2110663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng we need to mark it undefined to allow the below code to access is. */ 2111663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng INNER_REQUEST(VALGRIND_FREELIKE_BLOCK(base_p, a->rz_szB)); 2112663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng INNER_REQUEST(VALGRIND_MAKE_MEM_UNDEFINED(base_p, base_pszB_req)); 2113ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2114ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Block ptr for the block we are going to split. */ 2115ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown base_b = get_payload_block ( a, base_p ); 2116ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2117ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Pointer to the payload of the aligned block we are going to 2118ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return. This has to be suitably aligned. */ 2119ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown align_p = align_upwards ( base_b + 2 * overhead_szB_lo(a) 2120ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown + overhead_szB_hi(a), 2121ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown req_alignB ); 2122ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown align_b = get_payload_block(a, align_p); 2123ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2124ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* The block size of the fragment we will create. This must be big 2125ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown enough to actually create a fragment. */ 2126ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown frag_bszB = align_b - base_b; 2127ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2128ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(frag_bszB >= min_useful_bszB(a)); 2129ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2130ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* The actual payload size of the block we are going to split. */ 2131ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown base_pszB_act = get_pszB(a, base_b); 2132ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2133ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Create the fragment block, and put it back on the relevant free list. */ 2134ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mkFreeBlock ( a, base_b, frag_bszB, 2135ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown pszB_to_listNo(bszB_to_pszB(a, frag_bszB)) ); 2136ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (VG_(clo_profile_heap)) 2137ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown set_cc(base_b, "admin.frag-memalign-1"); 2138ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2139ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Create the aligned block. */ 2140ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mkInuseBlock ( a, align_b, 2141ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown base_p + base_pszB_act 2142ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown + overhead_szB_hi(a) - (UByte*)align_b ); 2143ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (VG_(clo_profile_heap)) 2144ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown set_cc(align_b, cc); 2145ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2146ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown /* Final sanity checks. */ 2147ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert( is_inuse_block(get_payload_block(a, align_p)) ); 2148ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2149ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(req_pszB <= get_pszB(a, get_payload_block(a, align_p))); 2150ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2151b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->stats__bytes_on_loan += get_pszB(a, get_payload_block(a, align_p)); 2152b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (a->stats__bytes_on_loan > a->stats__bytes_on_loan_max) { 2153b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov a->stats__bytes_on_loan_max = a->stats__bytes_on_loan; 2154b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 2155b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov /* a->stats__tot_blocks, a->stats__tot_bytes, a->stats__nsearches 2156b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov are updated by the call to VG_(arena_malloc) just a few lines 2157b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov above. So we don't need to update them here. */ 2158ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2159ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# ifdef DEBUG_MALLOC 2160ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown sanity_check_malloc_arena(aid); 2161ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# endif 2162ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2163ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert( (((Addr)align_p) % req_alignB) == 0 ); 2164ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2165663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng INNER_REQUEST(VALGRIND_MALLOCLIKE_BLOCK(align_p, 2166663860b1408516d02ebfcb3a9999a134e6cfb223Ben Cheng req_pszB, a->rz_szB, False)); 2167ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2168ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return align_p; 2169ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2170ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2171ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2172ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownSizeT VG_(arena_malloc_usable_size) ( ArenaId aid, void* ptr ) 2173ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2174ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Arena* a = arenaId_to_ArenaP(aid); 2175ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Block* b = get_payload_block(a, ptr); 2176ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return get_pszB(a, b); 2177ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2178ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2179ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2180ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Implementation of mallinfo(). There is no recent standard that defines 2181ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// the behavior of mallinfo(). The meaning of the fields in struct mallinfo 2182ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// is as follows: 2183ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// 2184ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// struct mallinfo { 2185ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// int arena; /* total space in arena */ 2186ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// int ordblks; /* number of ordinary blocks */ 2187ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// int smblks; /* number of small blocks */ 2188ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// int hblks; /* number of holding blocks */ 2189ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// int hblkhd; /* space in holding block headers */ 2190ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// int usmblks; /* space in small blocks in use */ 2191ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// int fsmblks; /* space in free small blocks */ 2192ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// int uordblks; /* space in ordinary blocks in use */ 2193ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// int fordblks; /* space in free ordinary blocks */ 2194ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// int keepcost; /* space penalty if keep option */ 2195ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// /* is used */ 2196ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// }; 2197ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// 2198ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// The glibc documentation about mallinfo (which is somewhat outdated) can 2199ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// be found here: 2200ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// http://www.gnu.org/software/libtool/manual/libc/Statistics-of-Malloc.html 2201ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// 2202ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// See also http://bugs.kde.org/show_bug.cgi?id=160956. 2203ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// 2204ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Regarding the implementation of VG_(mallinfo)(): we cannot return the 2205ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// whole struct as the library function does, because this is called by a 2206ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// client request. So instead we use a pointer to do call by reference. 2207ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid VG_(mallinfo) ( ThreadId tid, struct vg_mallinfo* mi ) 2208ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2209ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown UWord i, free_blocks, free_blocks_size; 2210ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Arena* a = arenaId_to_ArenaP(VG_AR_CLIENT); 2211ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2212ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // Traverse free list and calculate free blocks statistics. 2213ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // This may seem slow but glibc works the same way. 2214ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown free_blocks_size = free_blocks = 0; 2215ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < N_MALLOC_LISTS; i++) { 2216ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Block* b = a->freelist[i]; 2217ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (b == NULL) continue; 2218ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (;;) { 2219ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown free_blocks++; 2220ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown free_blocks_size += (UWord)get_pszB(a, b); 2221ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b = get_next_b(b); 2222ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (b == a->freelist[i]) break; 2223ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2224ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2225ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2226ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // We don't have fastbins so smblks & fsmblks are always 0. Also we don't 2227ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown // have a separate mmap allocator so set hblks & hblkhd to 0. 2228b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov mi->arena = a->stats__bytes_mmaped; 2229ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mi->ordblks = free_blocks + VG_(free_queue_length); 2230ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mi->smblks = 0; 2231ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mi->hblks = 0; 2232ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mi->hblkhd = 0; 2233ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mi->usmblks = 0; 2234ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mi->fsmblks = 0; 2235b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov mi->uordblks = a->stats__bytes_on_loan - VG_(free_queue_volume); 2236ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mi->fordblks = free_blocks_size + VG_(free_queue_volume); 2237ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mi->keepcost = 0; // may want some value in here 2238ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2239ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2240436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy IvanovSizeT VG_(arena_redzone_size) ( ArenaId aid ) 2241436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov{ 2242436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov ensure_mm_init (VG_AR_CLIENT); 2243436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov /* ensure_mm_init will call arena_init if not yet done. 2244436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov This then ensures that the arena redzone size is properly 2245436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov initialised. */ 2246436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov return arenaId_to_ArenaP(aid)->rz_szB; 2247436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov} 2248ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2249ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------------------------------------------------------*/ 2250ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--- Services layered on top of malloc/free. ---*/ 2251ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------------------------------------------------------*/ 2252ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2253436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovvoid* VG_(arena_calloc) ( ArenaId aid, const HChar* cc, 2254ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT nmemb, SizeT bytes_per_memb ) 2255ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2256ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT size; 2257436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov void* p; 2258ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2259ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown size = nmemb * bytes_per_memb; 2260ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(size >= nmemb && size >= bytes_per_memb);// check against overflow 2261ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2262ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown p = VG_(arena_malloc) ( aid, cc, size ); 2263ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2264436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov if (p != NULL) 2265436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov VG_(memset)(p, 0, size); 2266ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2267ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return p; 2268ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2269ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2270ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2271436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovvoid* VG_(arena_realloc) ( ArenaId aid, const HChar* cc, 2272ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown void* ptr, SizeT req_pszB ) 2273ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2274ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Arena* a; 2275ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown SizeT old_pszB; 2276436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov void* p_new; 2277ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Block* b; 2278ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2279ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ensure_mm_init(aid); 2280ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown a = arenaId_to_ArenaP(aid); 2281ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2282ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(req_pszB < MAX_PSZB); 2283ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2284ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (NULL == ptr) { 2285ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return VG_(arena_malloc)(aid, cc, req_pszB); 2286ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2287ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2288ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (req_pszB == 0) { 2289ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(arena_free)(aid, ptr); 2290ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return NULL; 2291ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2292ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2293ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown b = get_payload_block(a, ptr); 2294ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(blockSane(a, b)); 2295ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2296ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown vg_assert(is_inuse_block(b)); 2297ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown old_pszB = get_pszB(a, b); 2298ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2299ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (req_pszB <= old_pszB) { 2300ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ptr; 2301ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 2302ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2303ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown p_new = VG_(arena_malloc) ( aid, cc, req_pszB ); 2304ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2305ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(memcpy)(p_new, ptr, old_pszB); 2306ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2307ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown VG_(arena_free)(aid, ptr); 2308ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2309ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return p_new; 2310ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2311ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2312ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2313ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/* Inline just for the wrapper VG_(strdup) below */ 2314436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov__inline__ HChar* VG_(arena_strdup) ( ArenaId aid, const HChar* cc, 2315436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov const HChar* s ) 2316ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2317ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Int i; 2318ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown Int len; 2319436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov HChar* res; 2320ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2321ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (s == NULL) 2322ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return NULL; 2323ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2324ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown len = VG_(strlen)(s) + 1; 2325ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown res = VG_(arena_malloc) (aid, cc, len); 2326ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2327ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for (i = 0; i < len; i++) 2328ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown res[i] = s[i]; 2329ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return res; 2330ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2331ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2332436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovvoid* VG_(arena_perm_malloc) ( ArenaId aid, SizeT size, Int align ) 2333436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov{ 2334436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov Arena* a; 2335436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov 2336436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov ensure_mm_init(aid); 2337436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov a = arenaId_to_ArenaP(aid); 2338436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov 2339436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov align = align - 1; 2340436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov size = (size + align) & ~align; 2341436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov 2342436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov if (UNLIKELY(a->perm_malloc_current + size > a->perm_malloc_limit)) { 2343436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov // Get a superblock, but we will not insert it into the superblock list. 2344436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov // The superblock structure is not needed, so we will use the full 2345436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov // memory range of it. This superblock is however counted in the 2346436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov // mmaped statistics. 2347436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov Superblock* new_sb = newSuperblock (a, size); 2348436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov a->perm_malloc_limit = (Addr)&new_sb->payload_bytes[new_sb->n_payload_bytes - 1]; 2349436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov 2350436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov // We do not mind starting allocating from the beginning of the superblock 2351436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov // as afterwards, we "lose" it as a superblock. 2352436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov a->perm_malloc_current = (Addr)new_sb; 2353436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov } 2354436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov 2355436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov a->stats__perm_blocks += 1; 2356436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov a->stats__perm_bytes_on_loan += size; 2357436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov add_one_block_to_stats (a, size); 2358436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov 2359436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov a->perm_malloc_current += size; 2360436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov return (void*)(a->perm_malloc_current - size); 2361436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov} 2362ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2363ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------------------------------------------------------*/ 2364ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--- Tool-visible functions. ---*/ 2365ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*------------------------------------------------------------*/ 2366ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2367ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// All just wrappers to avoid exposing arenas to tools. 2368ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2369436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovvoid* VG_(malloc) ( const HChar* cc, SizeT nbytes ) 2370ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2371436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov return VG_(arena_malloc) ( VG_AR_CORE, cc, nbytes ); 2372ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2373ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2374ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownvoid VG_(free) ( void* ptr ) 2375ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2376436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov VG_(arena_free) ( VG_AR_CORE, ptr ); 2377ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2378ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2379436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovvoid* VG_(calloc) ( const HChar* cc, SizeT nmemb, SizeT bytes_per_memb ) 2380ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2381436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov return VG_(arena_calloc) ( VG_AR_CORE, cc, nmemb, bytes_per_memb ); 2382ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2383ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2384436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovvoid* VG_(realloc) ( const HChar* cc, void* ptr, SizeT size ) 2385ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2386436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov return VG_(arena_realloc) ( VG_AR_CORE, cc, ptr, size ); 2387ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2388ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2389436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy IvanovHChar* VG_(strdup) ( const HChar* cc, const HChar* s ) 2390ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2391436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov return VG_(arena_strdup) ( VG_AR_CORE, cc, s ); 2392ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2393ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2394ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown// Useful for querying user blocks. 2395ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff BrownSizeT VG_(malloc_usable_size) ( void* p ) 2396ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 2397ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return VG_(arena_malloc_usable_size)(VG_AR_CLIENT, p); 2398ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 2399ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2400436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovvoid* VG_(perm_malloc) ( SizeT size, Int align ) 2401436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov{ 2402436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov return VG_(arena_perm_malloc) ( VG_AR_CORE, size, align ); 2403436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov} 2404436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov 2405ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 2406ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--------------------------------------------------------------------*/ 2407ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--- end ---*/ 2408ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown/*--------------------------------------------------------------------*/ 2409