15821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Copyright (c) 2005, Google Inc. 25821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// All rights reserved. 35821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 45821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Redistribution and use in source and binary forms, with or without 55821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// modification, are permitted provided that the following conditions are 65821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// met: 75821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 85821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// * Redistributions of source code must retain the above copyright 95821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// notice, this list of conditions and the following disclaimer. 105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// * Redistributions in binary form must reproduce the above 115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// copyright notice, this list of conditions and the following disclaimer 125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// in the documentation and/or other materials provided with the 135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// distribution. 145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// * Neither the name of Google Inc. nor the names of its 155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// contributors may be used to endorse or promote products derived from 165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// this software without specific prior written permission. 175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// --- 315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Author: Sanjay Ghemawat <opensource@google.com> 325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// A malloc that uses a per-thread cache to satisfy small malloc requests. 345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// (The time for malloc/free of a small object drops from 300 ns to 50 ns.) 355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// See doc/tcmalloc.html for a high-level 375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// description of how this malloc works. 385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// SYNCHRONIZATION 405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 1. The thread-specific lists are accessed without acquiring any locks. 415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// This is safe because each such list is only accessed by one thread. 425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 2. We have a lock per central free-list, and hold it while manipulating 435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// the central free list for a particular size. 445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 3. The central page allocator is protected by "pageheap_lock". 455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 4. The pagemap (which maps from page-number to descriptor), 465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// can be read without holding any locks, and written while holding 475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// the "pageheap_lock". 485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 5. To improve performance, a subset of the information one can get 495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// from the pagemap is cached in a data structure, pagemap_cache_, 505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// that atomically reads and writes its entries. This cache can be 515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// read and written without locking. 525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// This multi-threaded access to the pagemap is safe for fairly 545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// subtle reasons. We basically assume that when an object X is 555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// allocated by thread A and deallocated by thread B, there must 565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// have been appropriate synchronization in the handoff of object 575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// X from thread A to thread B. The same logic applies to pagemap_cache_. 585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// THE PAGEID-TO-SIZECLASS CACHE 605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Hot PageID-to-sizeclass mappings are held by pagemap_cache_. If this cache 615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// returns 0 for a particular PageID then that means "no information," not that 625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// the sizeclass is 0. The cache may have stale information for pages that do 635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// not hold the beginning of any free()'able object. Staleness is eliminated 645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// in Populate() for pages with sizeclass > 0 objects, and in do_malloc() and 655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// do_memalign() for all other relevant pages. 665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// PAGEMAP 685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// ------- 695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Page map contains a mapping from page id to Span. 705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// If Span s occupies pages [p..q], 725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// pagemap[p] == s 735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// pagemap[q] == s 745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// pagemap[p+1..q-1] are undefined 755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// pagemap[p-1] and pagemap[q+1] are defined: 765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// NULL if the corresponding page is not yet in the address space. 775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Otherwise it points to a Span. This span may be free 785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// or allocated. If free, it is in one of pageheap's freelist. 795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// TODO: Bias reclamation to larger addresses 815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// TODO: implement mallinfo/mallopt 825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// TODO: Better testing 835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 9/28/2003 (new page-level allocator replaces ptmalloc2): 855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// * malloc/free of small objects goes from ~300 ns to ~50 ns. 865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// * allocation of a reasonably complicated struct 875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// goes from about 1100 ns to about 300 ns. 885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "config.h" 905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <gperftools/tcmalloc.h> 915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <errno.h> // for ENOMEM, EINVAL, errno 935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifdef HAVE_SYS_CDEFS_H 945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <sys/cdefs.h> // for __THROW 955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if defined HAVE_STDINT_H 975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <stdint.h> 985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#elif defined HAVE_INTTYPES_H 995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <inttypes.h> 1005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#else 1015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <sys/types.h> 1025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 1035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <stddef.h> // for size_t, NULL 1045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <stdlib.h> // for getenv 1055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <string.h> // for strcmp, memset, strlen, etc 1065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifdef HAVE_UNISTD_H 1075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <unistd.h> // for getpagesize, write, etc 1085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 1095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <algorithm> // for max, min 1105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <limits> // for numeric_limits 1115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <new> // for nothrow_t (ptr only), etc 1125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <vector> // for vector 1135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <gperftools/malloc_extension.h> 1155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <gperftools/malloc_hook.h> // for MallocHook 1165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/basictypes.h" // for int64 1175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/commandlineflags.h" // for RegisterFlagValidator, etc 1185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/dynamic_annotations.h" // for RunningOnValgrind 1195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/spinlock.h" // for SpinLockHolder 1205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "central_freelist.h" // for CentralFreeListPadded 1215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "common.h" // for StackTrace, kPageShift, etc 1225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "free_list.h" // for FL_Init 1235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "internal_logging.h" // for ASSERT, TCMalloc_Printer, etc 1245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "malloc_hook-inl.h" // for MallocHook::InvokeNewHook, etc 1255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "page_heap.h" // for PageHeap, PageHeap::Stats 1265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "page_heap_allocator.h" // for PageHeapAllocator 1275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "span.h" // for Span, DLL_Prepend, etc 1285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "stack_trace_table.h" // for StackTraceTable 1295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "static_vars.h" // for Static 1305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "system-alloc.h" // for DumpSystemAllocatorStats, etc 1315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "tcmalloc_guard.h" // for TCMallocGuard 1325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "thread_cache.h" // for ThreadCache 1335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if (defined(_WIN32) && !defined(__CYGWIN__) && !defined(__CYGWIN32__)) && !defined(WIN32_OVERRIDE_ALLOCATORS) 1355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# define WIN32_DO_PATCHING 1 1365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 1375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Some windows file somewhere (at least on cygwin) #define's small (!) 1395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// For instance, <windows.h> appears to have "#define small char". 1405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#undef small 1415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)using STL_NAMESPACE::max; 1432a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)using STL_NAMESPACE::min; 1445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)using STL_NAMESPACE::numeric_limits; 1455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)using STL_NAMESPACE::vector; 1465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "libc_override.h" 1485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// __THROW is defined in glibc (via <sys/cdefs.h>). It means, 1505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// counter-intuitively, "This function will never throw an exception." 1515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// It's an optional optimization tool, but we may need to use it to 1525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// match glibc prototypes. 1535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifndef __THROW // I guess we're not on a glibc system 1545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# define __THROW // __THROW is just an optimization, so ok to make it "" 1555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 1565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)using tcmalloc::AlignmentForSize; 1585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)using tcmalloc::kLog; 1595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)using tcmalloc::kCrash; 1605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)using tcmalloc::kCrashWithStats; 1615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)using tcmalloc::Log; 1625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)using tcmalloc::PageHeap; 1635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)using tcmalloc::PageHeapAllocator; 1645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)using tcmalloc::SizeMap; 1655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)using tcmalloc::Span; 1665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)using tcmalloc::StackTrace; 1675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)using tcmalloc::Static; 1685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)using tcmalloc::ThreadCache; 1695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// ---- Functions doing validation with an extra mark. 1715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static size_t ExcludeSpaceForMark(size_t size); 1725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void AddRoomForMark(size_t* size); 1735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void ExcludeMarkFromSize(size_t* new_size); 1745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void MarkAllocatedRegion(void* ptr); 1755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void ValidateAllocatedRegion(void* ptr, size_t cl); 1765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// ---- End validation functions. 1775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)DECLARE_int64(tcmalloc_sample_parameter); 1795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)DECLARE_double(tcmalloc_release_rate); 1805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// For windows, the printf we use to report large allocs is 1825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// potentially dangerous: it could cause a malloc that would cause an 1835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// infinite loop. So by default we set the threshold to a huge number 1845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// on windows, so this bad situation will never trigger. You can 1855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// always set TCMALLOC_LARGE_ALLOC_REPORT_THRESHOLD manually if you 1865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// want this functionality. 1875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifdef _WIN32 1885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)const int64 kDefaultLargeAllocReportThreshold = static_cast<int64>(1) << 62; 1895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#else 1905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)const int64 kDefaultLargeAllocReportThreshold = static_cast<int64>(1) << 30; 1915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 1925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)DEFINE_int64(tcmalloc_large_alloc_report_threshold, 1935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EnvToInt64("TCMALLOC_LARGE_ALLOC_REPORT_THRESHOLD", 1945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) kDefaultLargeAllocReportThreshold), 1955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "Allocations larger than this value cause a stack " 1965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "trace to be dumped to stderr. The threshold for " 1975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "dumping stack traces is increased by a factor of 1.125 " 1985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "every time we print a message so that the threshold " 1995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "automatically goes up by a factor of ~1000 every 60 " 2005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "messages. This bounds the amount of extra logging " 2015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "generated by this flag. Default value of this flag " 2025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "is very large and therefore you should see no extra " 2035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "logging unless the flag is overridden. Set to 0 to " 2045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "disable reporting entirely."); 2055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// We already declared these functions in tcmalloc.h, but we have to 2085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// declare them again to give them an ATTRIBUTE_SECTION: we want to 2095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// put all callers of MallocHook::Invoke* in this module into 2105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// ATTRIBUTE_SECTION(google_malloc) section, so that 2115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// MallocHook::GetCallerStackTrace can function accurately. 2125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" { 2135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* tc_malloc(size_t size) __THROW 2145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ATTRIBUTE_SECTION(google_malloc); 2155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void tc_free(void* ptr) __THROW 2165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ATTRIBUTE_SECTION(google_malloc); 2175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* tc_realloc(void* ptr, size_t size) __THROW 2185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ATTRIBUTE_SECTION(google_malloc); 2195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* tc_calloc(size_t nmemb, size_t size) __THROW 2205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ATTRIBUTE_SECTION(google_malloc); 2215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void tc_cfree(void* ptr) __THROW 2225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ATTRIBUTE_SECTION(google_malloc); 2235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* tc_memalign(size_t __alignment, size_t __size) __THROW 2255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ATTRIBUTE_SECTION(google_malloc); 2265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int tc_posix_memalign(void** ptr, size_t align, size_t size) __THROW 2275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ATTRIBUTE_SECTION(google_malloc); 2285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* tc_valloc(size_t __size) __THROW 2295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ATTRIBUTE_SECTION(google_malloc); 2305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* tc_pvalloc(size_t __size) __THROW 2315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ATTRIBUTE_SECTION(google_malloc); 2325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void tc_malloc_stats(void) __THROW 2345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ATTRIBUTE_SECTION(google_malloc); 2355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int tc_mallopt(int cmd, int value) __THROW 2365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ATTRIBUTE_SECTION(google_malloc); 2375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifdef HAVE_STRUCT_MALLINFO 2385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) struct mallinfo tc_mallinfo(void) __THROW 2395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ATTRIBUTE_SECTION(google_malloc); 2405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 2415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* tc_new(size_t size) 2435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ATTRIBUTE_SECTION(google_malloc); 2445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void tc_delete(void* p) __THROW 2455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ATTRIBUTE_SECTION(google_malloc); 2465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* tc_newarray(size_t size) 2475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ATTRIBUTE_SECTION(google_malloc); 2485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void tc_deletearray(void* p) __THROW 2495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ATTRIBUTE_SECTION(google_malloc); 2505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // And the nothrow variants of these: 2525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* tc_new_nothrow(size_t size, const std::nothrow_t&) __THROW 2535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ATTRIBUTE_SECTION(google_malloc); 2545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* tc_newarray_nothrow(size_t size, const std::nothrow_t&) __THROW 2555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ATTRIBUTE_SECTION(google_malloc); 2565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Surprisingly, standard C++ library implementations use a 2575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // nothrow-delete internally. See, eg: 2585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // http://www.dinkumware.com/manuals/?manual=compleat&page=new.html 2595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void tc_delete_nothrow(void* ptr, const std::nothrow_t&) __THROW 2605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ATTRIBUTE_SECTION(google_malloc); 2615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void tc_deletearray_nothrow(void* ptr, const std::nothrow_t&) __THROW 2625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ATTRIBUTE_SECTION(google_malloc); 2635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Some non-standard extensions that we support. 2655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // This is equivalent to 2675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // OS X: malloc_size() 2685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // glibc: malloc_usable_size() 2695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Windows: _msize() 2705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t tc_malloc_size(void* p) __THROW 2715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ATTRIBUTE_SECTION(google_malloc); 2725c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu 2735c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu void* tc_malloc_skip_new_handler(size_t size) 2745c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu ATTRIBUTE_SECTION(google_malloc); 2755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} // extern "C" 2765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// ----------------------- IMPLEMENTATION ------------------------------- 2795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int tc_new_mode = 0; // See tc_set_new_mode(). 2815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Routines such as free() and realloc() catch some erroneous pointers 2835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// passed to them, and invoke the below when they do. (An erroneous pointer 2845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// won't be caught if it's within a valid span or a stale span for which 2855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// the pagemap cache has a non-zero sizeclass.) This is a cheap (source-editing 2865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// required) kind of exception handling for these routines. 2875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)namespace { 2885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void InvalidFree(void* ptr) { 2895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Log(kCrash, __FILE__, __LINE__, "Attempt to free invalid pointer", ptr); 2905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)size_t InvalidGetSizeForRealloc(const void* old_ptr) { 2935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Log(kCrash, __FILE__, __LINE__, 2945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "Attempt to realloc invalid pointer", old_ptr); 2955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 0; 2965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)size_t InvalidGetAllocatedSize(const void* ptr) { 2995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Log(kCrash, __FILE__, __LINE__, 3005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "Attempt to get the size of an invalid pointer", ptr); 3015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 0; 3025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3032a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 3042a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)// For security reasons, we want to limit the size of allocations. 3052a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)// See crbug.com/169327. 3062a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)inline bool IsAllocSizePermitted(size_t alloc_size) { 3072a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) // Never allow an allocation larger than what can be indexed via an int. 3082a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) // Remove kPageSize to account for various rounding, padding and to have a 3092a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) // small margin. 3102a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return alloc_size <= ((std::numeric_limits<int>::max)() - kPageSize); 3112a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)} 3122a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 3135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} // unnamed namespace 3145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Extract interesting stats 3165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)struct TCMallocStats { 3175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint64_t thread_bytes; // Bytes in thread caches 3185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint64_t central_bytes; // Bytes in central cache 3195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint64_t transfer_bytes; // Bytes in central transfer cache 3205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint64_t metadata_bytes; // Bytes alloced for metadata 3215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint64_t metadata_unmapped_bytes; // Address space reserved for metadata 3225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // but is not committed. 3235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PageHeap::Stats pageheap; // Stats from page heap 3245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}; 3255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Get stats into "r". Also get per-size-class counts if class_count != NULL 3275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void ExtractStats(TCMallocStats* r, uint64_t* class_count, 3285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PageHeap::SmallSpanStats* small_spans, 3295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PageHeap::LargeSpanStats* large_spans) { 3305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) r->central_bytes = 0; 3315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) r->transfer_bytes = 0; 3325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for (int cl = 0; cl < kNumClasses; ++cl) { 3335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const int length = Static::central_cache()[cl].length(); 3345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const int tc_length = Static::central_cache()[cl].tc_length(); 3355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const size_t cache_overhead = Static::central_cache()[cl].OverheadBytes(); 3365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const size_t size = static_cast<uint64_t>( 3375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Static::sizemap()->ByteSizeForClass(cl)); 3385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) r->central_bytes += (size * length) + cache_overhead; 3395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) r->transfer_bytes += (size * tc_length); 3405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (class_count) class_count[cl] = length + tc_length; 3415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 3425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Add stats from per-thread heaps 3445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) r->thread_bytes = 0; 3455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) { // scope 3465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SpinLockHolder h(Static::pageheap_lock()); 3475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ThreadCache::GetThreadStats(&r->thread_bytes, class_count); 3485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) r->metadata_bytes = tcmalloc::metadata_system_bytes(); 3495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) r->metadata_unmapped_bytes = tcmalloc::metadata_unmapped_bytes(); 3505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) r->pageheap = Static::pageheap()->stats(); 3515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (small_spans != NULL) { 3525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Static::pageheap()->GetSmallSpanStats(small_spans); 3535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 3545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (large_spans != NULL) { 3555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Static::pageheap()->GetLargeSpanStats(large_spans); 3565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 3575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 3585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static double PagesToMiB(uint64_t pages) { 3615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (pages << kPageShift) / 1048576.0; 3625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// WRITE stats to "out" 3655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void DumpStats(TCMalloc_Printer* out, int level) { 3665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TCMallocStats stats; 3675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint64_t class_count[kNumClasses]; 3685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PageHeap::SmallSpanStats small; 3695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PageHeap::LargeSpanStats large; 3705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (level >= 2) { 3715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ExtractStats(&stats, class_count, &small, &large); 3725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } else { 3735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ExtractStats(&stats, NULL, NULL, NULL); 3745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 3755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static const double MiB = 1048576.0; 3775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const uint64_t physical_memory_used_by_metadata = 3795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) stats.metadata_bytes - stats.metadata_unmapped_bytes; 3805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const uint64_t unmapped_bytes = 3815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) stats.pageheap.unmapped_bytes + stats.metadata_unmapped_bytes; 3825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const uint64_t virtual_memory_used = (stats.pageheap.system_bytes 3845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) + stats.metadata_bytes); 3855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const uint64_t physical_memory_used = virtual_memory_used - unmapped_bytes; 3865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const uint64_t bytes_in_use_by_app = (physical_memory_used 3875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) - physical_memory_used_by_metadata 3885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) - stats.pageheap.free_bytes 3895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) - stats.central_bytes 3905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) - stats.transfer_bytes 3915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) - stats.thread_bytes); 3925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) out->printf( 3945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "WASTE: %7.1f MiB bytes in use\n" 3955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "WASTE: + %7.1f MiB committed but not used\n" 3965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "WASTE: ------------\n" 3975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "WASTE: = %7.1f MiB bytes committed\n" 3985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "WASTE: committed/used ratio of %f\n", 3995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) bytes_in_use_by_app / MiB, 4005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (stats.pageheap.committed_bytes - bytes_in_use_by_app) / MiB, 4015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) stats.pageheap.committed_bytes / MiB, 4025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) stats.pageheap.committed_bytes / static_cast<double>(bytes_in_use_by_app) 4035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ); 4045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifdef TCMALLOC_SMALL_BUT_SLOW 4055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) out->printf( 4065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "NOTE: SMALL MEMORY MODEL IS IN USE, PERFORMANCE MAY SUFFER.\n"); 4075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 4085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) out->printf( 4095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "------------------------------------------------\n" 4105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "MALLOC: %12" PRIu64 " (%7.1f MiB) Bytes in use by application\n" 4115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes in page heap freelist\n" 4125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes in central cache freelist\n" 4135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes in transfer cache freelist\n" 4145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes in thread cache freelists\n" 4155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "MALLOC: ------------\n" 4165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "MALLOC: = %12" PRIu64 " (%7.1f MiB) Bytes committed\n" 4175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes in malloc metadata\n" 4185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "MALLOC: ------------\n" 4195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "MALLOC: = %12" PRIu64 " (%7.1f MiB) Actual memory used (physical + swap)\n" 4205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes released to OS (aka unmapped)\n" 4215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "MALLOC: ------------\n" 4225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "MALLOC: = %12" PRIu64 " (%7.1f MiB) Virtual address space used\n" 4235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "MALLOC:\n" 4245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "MALLOC: %12" PRIu64 " Spans in use\n" 4255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "MALLOC: %12" PRIu64 " Thread heaps in use\n" 4265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "MALLOC: %12" PRIu64 " Tcmalloc page size\n" 4275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "------------------------------------------------\n" 4285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "Call ReleaseFreeMemory() to release freelist memory to the OS" 4295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) " (via madvise()).\n" 4305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "Bytes released to the OS take up virtual address space" 4315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) " but no physical memory.\n", 4325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) bytes_in_use_by_app, bytes_in_use_by_app / MiB, 4335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) stats.pageheap.free_bytes, stats.pageheap.free_bytes / MiB, 4345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) stats.central_bytes, stats.central_bytes / MiB, 4355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) stats.transfer_bytes, stats.transfer_bytes / MiB, 4365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) stats.thread_bytes, stats.thread_bytes / MiB, 4375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) stats.pageheap.committed_bytes, stats.pageheap.committed_bytes / MiB, 4385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) physical_memory_used_by_metadata , physical_memory_used_by_metadata / MiB, 4395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) physical_memory_used, physical_memory_used / MiB, 4405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) unmapped_bytes, unmapped_bytes / MiB, 4415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) virtual_memory_used, virtual_memory_used / MiB, 4425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint64_t(Static::span_allocator()->inuse()), 4435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint64_t(ThreadCache::HeapsInUse()), 4445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint64_t(kPageSize)); 4455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (level >= 2) { 4475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) out->printf("------------------------------------------------\n"); 4485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) out->printf("Size class breakdown\n"); 4495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) out->printf("------------------------------------------------\n"); 4505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint64_t cumulative = 0; 4515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for (int cl = 0; cl < kNumClasses; ++cl) { 4525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (class_count[cl] > 0) { 4535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint64_t class_bytes = 4545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) class_count[cl] * Static::sizemap()->ByteSizeForClass(cl); 4555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cumulative += class_bytes; 4565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) out->printf("class %3d [ %8" PRIuS " bytes ] : " 4575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "%8" PRIu64 " objs; %5.1f MiB; %5.1f cum MiB\n", 4585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cl, Static::sizemap()->ByteSizeForClass(cl), 4595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) class_count[cl], 4605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) class_bytes / MiB, 4615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cumulative / MiB); 4625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // append page heap info 4665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nonempty_sizes = 0; 4675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for (int s = 0; s < kMaxPages; s++) { 4685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (small.normal_length[s] + small.returned_length[s] > 0) { 4695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nonempty_sizes++; 4705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) out->printf("------------------------------------------------\n"); 4735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) out->printf("PageHeap: %d sizes; %6.1f MiB free; %6.1f MiB unmapped\n", 4745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nonempty_sizes, stats.pageheap.free_bytes / MiB, 4755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) stats.pageheap.unmapped_bytes / MiB); 4765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) out->printf("------------------------------------------------\n"); 4775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint64_t total_normal = 0; 4785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint64_t total_returned = 0; 4795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for (int s = 0; s < kMaxPages; s++) { 4805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const int n_length = small.normal_length[s]; 4815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const int r_length = small.returned_length[s]; 4825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (n_length + r_length > 0) { 4835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint64_t n_pages = s * n_length; 4845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint64_t r_pages = s * r_length; 4855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) total_normal += n_pages; 4865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) total_returned += r_pages; 4875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) out->printf("%6u pages * %6u spans ~ %6.1f MiB; %6.1f MiB cum" 4885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "; unmapped: %6.1f MiB; %6.1f MiB cum\n", 4895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) s, 4905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (n_length + r_length), 4915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PagesToMiB(n_pages + r_pages), 4925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PagesToMiB(total_normal + total_returned), 4935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PagesToMiB(r_pages), 4945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PagesToMiB(total_returned)); 4955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) total_normal += large.normal_pages; 4995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) total_returned += large.returned_pages; 5005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) out->printf(">255 large * %6u spans ~ %6.1f MiB; %6.1f MiB cum" 5015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "; unmapped: %6.1f MiB; %6.1f MiB cum\n", 5025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static_cast<unsigned int>(large.spans), 5035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PagesToMiB(large.normal_pages + large.returned_pages), 5045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PagesToMiB(total_normal + total_returned), 5055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PagesToMiB(large.returned_pages), 5065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PagesToMiB(total_returned)); 5075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 5085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 5095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void PrintStats(int level) { 5115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const int kBufferSize = 16 << 10; 5125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char* buffer = new char[kBufferSize]; 5135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TCMalloc_Printer printer(buffer, kBufferSize); 5145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) DumpStats(&printer, level); 5155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) write(STDERR_FILENO, buffer, strlen(buffer)); 5165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) delete[] buffer; 5175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 5185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void** DumpHeapGrowthStackTraces() { 5205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Count how much space we need 5215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int needed_slots = 0; 5225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) { 5235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SpinLockHolder h(Static::pageheap_lock()); 5245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for (StackTrace* t = Static::growth_stacks(); 5255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) t != NULL; 5265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) t = reinterpret_cast<StackTrace*>( 5275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) t->stack[tcmalloc::kMaxStackDepth-1])) { 5285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) needed_slots += 3 + t->depth; 5295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 5305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) needed_slots += 100; // Slop in case list grows 5315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) needed_slots += needed_slots/8; // An extra 12.5% slop 5325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 5335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void** result = new void*[needed_slots]; 5355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (result == NULL) { 5365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Log(kLog, __FILE__, __LINE__, 5375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "tcmalloc: allocation failed for stack trace slots", 5385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) needed_slots * sizeof(*result)); 5395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return NULL; 5405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 5415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SpinLockHolder h(Static::pageheap_lock()); 5435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int used_slots = 0; 5445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for (StackTrace* t = Static::growth_stacks(); 5455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) t != NULL; 5465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) t = reinterpret_cast<StackTrace*>( 5475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) t->stack[tcmalloc::kMaxStackDepth-1])) { 5485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ASSERT(used_slots < needed_slots); // Need to leave room for terminator 5495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (used_slots + 3 + t->depth >= needed_slots) { 5505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // No more room 5515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) break; 5525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 5535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) result[used_slots+0] = reinterpret_cast<void*>(static_cast<uintptr_t>(1)); 5555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) result[used_slots+1] = reinterpret_cast<void*>(t->size); 5565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) result[used_slots+2] = reinterpret_cast<void*>(t->depth); 5575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for (int d = 0; d < t->depth; d++) { 5585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) result[used_slots+3+d] = t->stack[d]; 5595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 5605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) used_slots += 3 + t->depth; 5615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 5625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) result[used_slots] = reinterpret_cast<void*>(static_cast<uintptr_t>(0)); 5635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return result; 5645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 5655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void IterateOverRanges(void* arg, MallocExtension::RangeFunction func) { 5675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PageID page = 1; // Some code may assume that page==0 is never used 5685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) bool done = false; 5695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while (!done) { 5705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Accumulate a small number of ranges in a local buffer 5715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static const int kNumRanges = 16; 5725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static base::MallocRange ranges[kNumRanges]; 5735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int n = 0; 5745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) { 5755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SpinLockHolder h(Static::pageheap_lock()); 5765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while (n < kNumRanges) { 5775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (!Static::pageheap()->GetNextRange(page, &ranges[n])) { 5785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) done = true; 5795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) break; 5805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } else { 5815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uintptr_t limit = ranges[n].address + ranges[n].length; 5825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) page = (limit + kPageSize - 1) >> kPageShift; 5835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) n++; 5845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 5855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 5865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 5875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for (int i = 0; i < n; i++) { 5895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (*func)(arg, &ranges[i]); 5905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 5915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 5925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 5935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// TCMalloc's support for extra malloc interfaces 5955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class TCMallocImplementation : public MallocExtension { 5965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) private: 5975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // ReleaseToSystem() might release more than the requested bytes because 5985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // the page heap releases at the span granularity, and spans are of wildly 5995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // different sizes. This member keeps track of the extra bytes bytes 6005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // released so that the app can periodically call ReleaseToSystem() to 6015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // release memory at a constant rate. 6025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // NOTE: Protected by Static::pageheap_lock(). 6035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t extra_bytes_released_; 6045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) public: 6065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TCMallocImplementation() 6075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : extra_bytes_released_(0) { 6085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 6095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) virtual void GetStats(char* buffer, int buffer_length) { 6115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ASSERT(buffer_length > 0); 6125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TCMalloc_Printer printer(buffer, buffer_length); 6135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Print level one stats unless lots of space is available 6155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (buffer_length < 10000) { 6165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) DumpStats(&printer, 1); 6175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } else { 6185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) DumpStats(&printer, 2); 6195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 6205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 6215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // We may print an extra, tcmalloc-specific warning message here. 6235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) virtual void GetHeapSample(MallocExtensionWriter* writer) { 6245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (FLAGS_tcmalloc_sample_parameter == 0) { 6255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char* const kWarningMsg = 6265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "%warn\n" 6275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "%warn This heap profile does not have any data in it, because\n" 6285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "%warn the application was run with heap sampling turned off.\n" 6295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "%warn To get useful data from GetHeapSample(), you must\n" 6305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "%warn set the environment variable TCMALLOC_SAMPLE_PARAMETER to\n" 6315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "%warn a positive sampling period, such as 524288.\n" 6325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "%warn\n"; 6335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) writer->append(kWarningMsg, strlen(kWarningMsg)); 6345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 6355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocExtension::GetHeapSample(writer); 6365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 6375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) virtual void** ReadStackTraces(int* sample_period) { 6395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tcmalloc::StackTraceTable table; 6405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) { 6415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SpinLockHolder h(Static::pageheap_lock()); 6425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Span* sampled = Static::sampled_objects(); 6435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for (Span* s = sampled->next; s != sampled; s = s->next) { 6445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) table.AddTrace(*reinterpret_cast<StackTrace*>(s->objects)); 6455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 6465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 6475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *sample_period = ThreadCache::GetCache()->GetSamplePeriod(); 6485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return table.ReadStackTracesAndClear(); // grabs and releases pageheap_lock 6495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 6505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) virtual void** ReadHeapGrowthStackTraces() { 6525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return DumpHeapGrowthStackTraces(); 6535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 6545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) virtual void Ranges(void* arg, RangeFunction func) { 6565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) IterateOverRanges(arg, func); 6575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 6585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) virtual bool GetNumericProperty(const char* name, size_t* value) { 6605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ASSERT(name != NULL); 6615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (strcmp(name, "generic.current_allocated_bytes") == 0) { 6635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TCMallocStats stats; 6645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ExtractStats(&stats, NULL, NULL, NULL); 6655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *value = stats.pageheap.system_bytes 6665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) - stats.thread_bytes 6675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) - stats.central_bytes 6685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) - stats.transfer_bytes 6695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) - stats.pageheap.free_bytes 6705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) - stats.pageheap.unmapped_bytes; 6715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return true; 6725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 6735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (strcmp(name, "generic.heap_size") == 0) { 6755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TCMallocStats stats; 6765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ExtractStats(&stats, NULL, NULL, NULL); 6775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *value = stats.pageheap.system_bytes; 6785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return true; 6795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 6805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (strcmp(name, "tcmalloc.slack_bytes") == 0) { 6825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Kept for backwards compatibility. Now defined externally as: 6835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // pageheap_free_bytes + pageheap_unmapped_bytes. 6845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SpinLockHolder l(Static::pageheap_lock()); 6855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PageHeap::Stats stats = Static::pageheap()->stats(); 6865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *value = stats.free_bytes + stats.unmapped_bytes; 6875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return true; 6885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 6895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (strcmp(name, "tcmalloc.pageheap_free_bytes") == 0) { 6915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SpinLockHolder l(Static::pageheap_lock()); 6925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *value = Static::pageheap()->stats().free_bytes; 6935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return true; 6945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 6955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (strcmp(name, "tcmalloc.pageheap_unmapped_bytes") == 0) { 6975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SpinLockHolder l(Static::pageheap_lock()); 6985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *value = Static::pageheap()->stats().unmapped_bytes; 6995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return true; 7005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 7015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (strcmp(name, "tcmalloc.max_total_thread_cache_bytes") == 0) { 7035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SpinLockHolder l(Static::pageheap_lock()); 7045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *value = ThreadCache::overall_thread_cache_size(); 7055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return true; 7065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 7075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (strcmp(name, "tcmalloc.current_total_thread_cache_bytes") == 0) { 7095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TCMallocStats stats; 7105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ExtractStats(&stats, NULL, NULL, NULL); 7115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *value = stats.thread_bytes; 7125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return true; 7135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 7145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return false; 7165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 7175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) virtual bool SetNumericProperty(const char* name, size_t value) { 7195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ASSERT(name != NULL); 7205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (strcmp(name, "tcmalloc.max_total_thread_cache_bytes") == 0) { 7225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SpinLockHolder l(Static::pageheap_lock()); 7235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ThreadCache::set_overall_thread_cache_size(value); 7245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return true; 7255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 7265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return false; 7285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 7295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) virtual void MarkThreadIdle() { 7315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ThreadCache::BecomeIdle(); 7325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 7335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) virtual void MarkThreadBusy(); // Implemented below 7355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) virtual SysAllocator* GetSystemAllocator() { 7375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SpinLockHolder h(Static::pageheap_lock()); 7385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return sys_alloc; 7395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 7405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) virtual void SetSystemAllocator(SysAllocator* alloc) { 7425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SpinLockHolder h(Static::pageheap_lock()); 7435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sys_alloc = alloc; 7445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 7455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) virtual void ReleaseToSystem(size_t num_bytes) { 7475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SpinLockHolder h(Static::pageheap_lock()); 7485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (num_bytes <= extra_bytes_released_) { 7495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // We released too much on a prior call, so don't release any 7505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // more this time. 7515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) extra_bytes_released_ = extra_bytes_released_ - num_bytes; 7525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return; 7535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 7545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) num_bytes = num_bytes - extra_bytes_released_; 7555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // num_bytes might be less than one page. If we pass zero to 7565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // ReleaseAtLeastNPages, it won't do anything, so we release a whole 7575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // page now and let extra_bytes_released_ smooth it out over time. 7585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Length num_pages = max<Length>(num_bytes >> kPageShift, 1); 7595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t bytes_released = Static::pageheap()->ReleaseAtLeastNPages( 7605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) num_pages) << kPageShift; 7615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (bytes_released > num_bytes) { 7625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) extra_bytes_released_ = bytes_released - num_bytes; 7635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } else { 7645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // The PageHeap wasn't able to release num_bytes. Don't try to 7655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // compensate with a big release next time. Specifically, 7665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // ReleaseFreeMemory() calls ReleaseToSystem(LONG_MAX). 7675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) extra_bytes_released_ = 0; 7685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 7695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 7705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) virtual void SetMemoryReleaseRate(double rate) { 7725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FLAGS_tcmalloc_release_rate = rate; 7735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 7745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) virtual double GetMemoryReleaseRate() { 7765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return FLAGS_tcmalloc_release_rate; 7775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 7785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) virtual size_t GetEstimatedAllocatedSize(size_t size) { 7795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (size <= kMaxSize) { 7805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const size_t cl = Static::sizemap()->SizeClass(size); 7815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const size_t alloc_size = Static::sizemap()->ByteSizeForClass(cl); 7825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return alloc_size; 7835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } else { 7845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return tcmalloc::pages(size) << kPageShift; 7855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 7865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 7875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // This just calls GetSizeWithCallback, but because that's in an 7895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // unnamed namespace, we need to move the definition below it in the 7905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // file. 7915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) virtual size_t GetAllocatedSize(const void* ptr); 7925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // This duplicates some of the logic in GetSizeWithCallback, but is 7945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // faster. This is important on OS X, where this function is called 7955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // on every allocation operation. 7965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) virtual Ownership GetOwnership(const void* ptr) { 7975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; 7985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // The rest of tcmalloc assumes that all allocated pointers use at 7995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // most kAddressBits bits. If ptr doesn't, then it definitely 8005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // wasn't alloacted by tcmalloc. 8015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if ((p >> (kAddressBits - kPageShift)) > 0) { 8025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return kNotOwned; 8035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 8045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t cl = Static::pageheap()->GetSizeClassIfCached(p); 8055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (cl != 0) { 8065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return kOwned; 8075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 8085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const Span *span = Static::pageheap()->GetDescriptor(p); 8095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return span ? kOwned : kNotOwned; 8105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 8115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) virtual void GetFreeListSizes(vector<MallocExtension::FreeListInfo>* v) { 8135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static const char* kCentralCacheType = "tcmalloc.central"; 8145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static const char* kTransferCacheType = "tcmalloc.transfer"; 8155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static const char* kThreadCacheType = "tcmalloc.thread"; 8165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static const char* kPageHeapType = "tcmalloc.page"; 8175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static const char* kPageHeapUnmappedType = "tcmalloc.page_unmapped"; 8185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static const char* kLargeSpanType = "tcmalloc.large"; 8195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static const char* kLargeUnmappedSpanType = "tcmalloc.large_unmapped"; 8205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) v->clear(); 8225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // central class information 8245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int64 prev_class_size = 0; 8255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for (int cl = 1; cl < kNumClasses; ++cl) { 8265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t class_size = Static::sizemap()->ByteSizeForClass(cl); 8275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocExtension::FreeListInfo i; 8285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) i.min_object_size = prev_class_size + 1; 8295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) i.max_object_size = class_size; 8305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) i.total_bytes_free = 8315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Static::central_cache()[cl].length() * class_size; 8325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) i.type = kCentralCacheType; 8335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) v->push_back(i); 8345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // transfer cache 8365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) i.total_bytes_free = 8375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Static::central_cache()[cl].tc_length() * class_size; 8385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) i.type = kTransferCacheType; 8395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) v->push_back(i); 8405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) prev_class_size = Static::sizemap()->ByteSizeForClass(cl); 8425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 8435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Add stats from per-thread heaps 8455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint64_t class_count[kNumClasses]; 8465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memset(class_count, 0, sizeof(class_count)); 8475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) { 8485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SpinLockHolder h(Static::pageheap_lock()); 8495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint64_t thread_bytes = 0; 8505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ThreadCache::GetThreadStats(&thread_bytes, class_count); 8515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 8525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) prev_class_size = 0; 8545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for (int cl = 1; cl < kNumClasses; ++cl) { 8555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocExtension::FreeListInfo i; 8565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) i.min_object_size = prev_class_size + 1; 8575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) i.max_object_size = Static::sizemap()->ByteSizeForClass(cl); 8585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) i.total_bytes_free = 8595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) class_count[cl] * Static::sizemap()->ByteSizeForClass(cl); 8605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) i.type = kThreadCacheType; 8615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) v->push_back(i); 8625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 8635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // append page heap info 8655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PageHeap::SmallSpanStats small; 8665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PageHeap::LargeSpanStats large; 8675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) { 8685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SpinLockHolder h(Static::pageheap_lock()); 8695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Static::pageheap()->GetSmallSpanStats(&small); 8705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Static::pageheap()->GetLargeSpanStats(&large); 8715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 8725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // large spans: mapped 8745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocExtension::FreeListInfo span_info; 8755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) span_info.type = kLargeSpanType; 8765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) span_info.max_object_size = (numeric_limits<size_t>::max)(); 8775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) span_info.min_object_size = kMaxPages << kPageShift; 8785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) span_info.total_bytes_free = large.normal_pages << kPageShift; 8795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) v->push_back(span_info); 8805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // large spans: unmapped 8825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) span_info.type = kLargeUnmappedSpanType; 8835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) span_info.total_bytes_free = large.returned_pages << kPageShift; 8845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) v->push_back(span_info); 8855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // small spans 8875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for (int s = 1; s < kMaxPages; s++) { 8885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocExtension::FreeListInfo i; 8895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) i.max_object_size = (s << kPageShift); 8905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) i.min_object_size = ((s - 1) << kPageShift); 8915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) i.type = kPageHeapType; 8935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) i.total_bytes_free = (s << kPageShift) * small.normal_length[s]; 8945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) v->push_back(i); 8955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) i.type = kPageHeapUnmappedType; 8975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) i.total_bytes_free = (s << kPageShift) * small.returned_length[s]; 8985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) v->push_back(i); 8995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 9005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 9015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}; 9025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// The constructor allocates an object to ensure that initialization 9045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// runs before main(), and therefore we do not have a chance to become 9055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// multi-threaded before initialization. We also create the TSD key 9065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// here. Presumably by the time this constructor runs, glibc is in 9075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// good enough shape to handle pthread_key_create(). 9085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 9095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// The constructor also takes the opportunity to tell STL to use 9105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// tcmalloc. We want to do this early, before construct time, so 9115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// all user STL allocations go through tcmalloc (which works really 9125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// well for STL). 9135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 9145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// The destructor prints stats when the program exits. 9155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int tcmallocguard_refcount = 0; // no lock needed: runs before main() 9165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)TCMallocGuard::TCMallocGuard() { 9175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (tcmallocguard_refcount++ == 0) { 9185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifdef HAVE_TLS // this is true if the cc/ld/libc combo support TLS 9195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Check whether the kernel also supports TLS (needs to happen at runtime) 9205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tcmalloc::CheckIfKernelSupportsTLS(); 9215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 9225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ReplaceSystemAlloc(); // defined in libc_override_*.h 9235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tc_free(tc_malloc(1)); 9245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ThreadCache::InitTSD(); 9255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tc_free(tc_malloc(1)); 9265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Either we, or debugallocation.cc, or valgrind will control memory 9275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // management. We register our extension if we're the winner. 9285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifdef TCMALLOC_USING_DEBUGALLOCATION 9295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Let debugallocation register its extension. 9305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#else 9315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (RunningOnValgrind()) { 9325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Let Valgrind uses its own malloc (so don't register our extension). 9335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } else { 9345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocExtension::Register(new TCMallocImplementation); 9355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 9365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 9375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 9385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 9395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)TCMallocGuard::~TCMallocGuard() { 9415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (--tcmallocguard_refcount == 0) { 9425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char* env = getenv("MALLOCSTATS"); 9435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (env != NULL) { 9445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int level = atoi(env); 9455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (level < 1) level = 1; 9465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PrintStats(level); 9475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 9485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 9495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 9505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifndef WIN32_OVERRIDE_ALLOCATORS 9515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static TCMallocGuard module_enter_exit_hook; 9525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 9535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)//------------------------------------------------------------------- 9555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Helpers for the exported routines below 9565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)//------------------------------------------------------------------- 9575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static inline bool CheckCachedSizeClass(void *ptr) { 9595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; 9605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t cached_value = Static::pageheap()->GetSizeClassIfCached(p); 9615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return cached_value == 0 || 9625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cached_value == Static::pageheap()->GetDescriptor(p)->sizeclass; 9635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 9645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static inline void* CheckMallocResult(void *result) { 9665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ASSERT(result == NULL || CheckCachedSizeClass(result)); 9675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MarkAllocatedRegion(result); 9685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return result; 9695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 9705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static inline void* SpanToMallocResult(Span *span) { 9725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Static::pageheap()->CacheSizeClass(span->start, 0); 9735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 9745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CheckMallocResult(reinterpret_cast<void*>(span->start << kPageShift)); 9755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 9765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void* DoSampledAllocation(size_t size) { 9785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Grab the stack trace outside the heap lock 9795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) StackTrace tmp; 9805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tmp.depth = GetStackTrace(tmp.stack, tcmalloc::kMaxStackDepth, 1); 9815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tmp.size = size; 9825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SpinLockHolder h(Static::pageheap_lock()); 9845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Allocate span 9855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Span *span = Static::pageheap()->New(tcmalloc::pages(size == 0 ? 1 : size)); 9865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (span == NULL) { 9875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return NULL; 9885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 9895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Allocate stack trace 9915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) StackTrace *stack = Static::stacktrace_allocator()->New(); 9925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (stack == NULL) { 9935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Sampling failed because of lack of memory 9945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return span; 9955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 9965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *stack = tmp; 9975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) span->sample = 1; 9985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) span->objects = stack; 9995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tcmalloc::DLL_Prepend(Static::sampled_objects(), span); 10005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SpanToMallocResult(span); 10025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 10035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)namespace { 10055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Copy of FLAGS_tcmalloc_large_alloc_report_threshold with 10075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// automatic increases factored in. 10085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int64_t large_alloc_threshold = 10095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (kPageSize > FLAGS_tcmalloc_large_alloc_report_threshold 10105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ? kPageSize : FLAGS_tcmalloc_large_alloc_report_threshold); 10115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void ReportLargeAlloc(Length num_pages, void* result) { 10135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) StackTrace stack; 10145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) stack.depth = GetStackTrace(stack.stack, tcmalloc::kMaxStackDepth, 1); 10155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static const int N = 1000; 10175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char buffer[N]; 10185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TCMalloc_Printer printer(buffer, N); 1019558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch printer.printf("tcmalloc: large alloc %" PRIu64 " bytes == %p @ ", 10205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static_cast<uint64>(num_pages) << kPageShift, 10215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) result); 10225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for (int i = 0; i < stack.depth; i++) { 10235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) printer.printf(" %p", stack.stack[i]); 10245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 10255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) printer.printf("\n"); 10265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) write(STDERR_FILENO, buffer, strlen(buffer)); 10275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 10285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void* cpp_alloc(size_t size, bool nothrow); 10305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void* do_malloc(size_t size); 10315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// TODO(willchan): Investigate whether or not inlining this much is harmful to 10335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// performance. 10345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// This is equivalent to do_malloc() except when tc_new_mode is set to true. 10355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Otherwise, it will run the std::new_handler if set. 10365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void* do_malloc_or_cpp_alloc(size_t size) { 10375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return tc_new_mode ? cpp_alloc(size, true) : do_malloc(size); 10385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 10395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void* cpp_memalign(size_t align, size_t size); 10415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void* do_memalign(size_t align, size_t size); 10425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void* do_memalign_or_cpp_memalign(size_t align, size_t size) { 10445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return tc_new_mode ? cpp_memalign(align, size) : do_memalign(align, size); 10455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 10465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Must be called with the page lock held. 10485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline bool should_report_large(Length num_pages) { 10495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const int64 threshold = large_alloc_threshold; 10505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (threshold > 0 && num_pages >= (threshold >> kPageShift)) { 10515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Increase the threshold by 1/8 every time we generate a report. 10525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // We cap the threshold at 8GiB to avoid overflow problems. 10535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) large_alloc_threshold = (threshold + threshold/8 < 8ll<<30 10545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ? threshold + threshold/8 : 8ll<<30); 10555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return true; 10565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 10575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return false; 10585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 10595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Helper for do_malloc(). 10615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void* do_malloc_pages(ThreadCache* heap, size_t size) { 10625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* result; 10635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) bool report_large; 10645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Length num_pages = tcmalloc::pages(size); 10665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size = num_pages << kPageShift; 10675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10682a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) // Chromium profiling. Measurements in March 2013 suggest this 10692a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) // imposes a small enough runtime cost that there's no reason to 10702a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) // try to optimize it. 10712a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) heap->AddToByteAllocatedTotal(size); 10725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if ((FLAGS_tcmalloc_sample_parameter > 0) && heap->SampleAllocation(size)) { 10745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) result = DoSampledAllocation(size); 10755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SpinLockHolder h(Static::pageheap_lock()); 10775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) report_large = should_report_large(num_pages); 10785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } else { 10795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SpinLockHolder h(Static::pageheap_lock()); 10805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Span* span = Static::pageheap()->New(num_pages); 10815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) result = (span == NULL ? NULL : SpanToMallocResult(span)); 10825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) report_large = should_report_large(num_pages); 10835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 10845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (report_large) { 10865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ReportLargeAlloc(num_pages, result); 10875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 10885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return result; 10895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 10905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void* do_malloc(size_t size) { 10925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AddRoomForMark(&size); 10935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* ret = NULL; 10955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // The following call forces module initialization 10975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ThreadCache* heap = ThreadCache::GetCache(); 10982a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if (size <= kMaxSize && IsAllocSizePermitted(size)) { 10995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t cl = Static::sizemap()->SizeClass(size); 11005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size = Static::sizemap()->class_to_size(cl); 11015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11022a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) // Chromium profiling. Measurements in March 2013 suggest this 11032a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) // imposes a small enough runtime cost that there's no reason to 11042a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) // try to optimize it. 11052a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) heap->AddToByteAllocatedTotal(size); 11065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11072a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if ((FLAGS_tcmalloc_sample_parameter > 0) && 11082a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) heap->SampleAllocation(size)) { 11095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ret = DoSampledAllocation(size); 11105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MarkAllocatedRegion(ret); 11115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } else { 11125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // The common case, and also the simplest. This just pops the 11135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // size-appropriate freelist, after replenishing it if it's empty. 11145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ret = CheckMallocResult(heap->Allocate(size, cl)); 11155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 11162a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) } else if (IsAllocSizePermitted(size)) { 11175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ret = do_malloc_pages(heap, size); 11185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MarkAllocatedRegion(ret); 11195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 11205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (ret == NULL) errno = ENOMEM; 11212a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) ASSERT(IsAllocSizePermitted(size) || ret == NULL); 11225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ret; 11235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 11245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void* do_calloc(size_t n, size_t elem_size) { 11265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Overflow check 11275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const size_t size = n * elem_size; 11285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (elem_size != 0 && size / elem_size != n) return NULL; 11295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* result = do_malloc_or_cpp_alloc(size); 11315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (result != NULL) { 11325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memset(result, 0, size); 11335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 11345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return result; 11355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 11365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static inline ThreadCache* GetCacheIfPresent() { 11385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* const p = ThreadCache::GetCacheIfPresent(); 11395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return reinterpret_cast<ThreadCache*>(p); 11405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 11415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// This lets you call back to a given function pointer if ptr is invalid. 11435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// It is used primarily by windows code which wants a specialized callback. 11445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void do_free_with_callback(void* ptr, void (*invalid_free_fn)(void*)) { 11455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (ptr == NULL) return; 11465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (Static::pageheap() == NULL) { 11475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // We called free() before malloc(). This can occur if the 11485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // (system) malloc() is called before tcmalloc is loaded, and then 11495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // free() is called after tcmalloc is loaded (and tc_free has 11505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // replaced free), but before the global constructor has run that 11515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // sets up the tcmalloc data structures. 11525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (*invalid_free_fn)(ptr); // Decide how to handle the bad free request 11535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return; 11545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 11555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; 11565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Span* span = NULL; 11575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t cl = Static::pageheap()->GetSizeClassIfCached(p); 11585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (cl == 0) { 11605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) span = Static::pageheap()->GetDescriptor(p); 11615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (!span) { 11625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // span can be NULL because the pointer passed in is invalid 11635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // (not something returned by malloc or friends), or because the 11645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // pointer was allocated with some other allocator besides 11655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // tcmalloc. The latter can happen if tcmalloc is linked in via 11665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // a dynamic library, but is not listed last on the link line. 11675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // In that case, libraries after it on the link line will 11685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // allocate with libc malloc, but free with tcmalloc's free. 11695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (*invalid_free_fn)(ptr); // Decide how to handle the bad free request 11705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return; 11715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 11725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cl = span->sizeclass; 11735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Static::pageheap()->CacheSizeClass(p, cl); 11745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 11755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (cl == 0) { 11765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Check to see if the object is in use. 11775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK_CONDITION_PRINT(span->location == Span::IN_USE, 11785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "Object was not in-use"); 11795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK_CONDITION_PRINT( 11815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) span->start << kPageShift == reinterpret_cast<uintptr_t>(ptr), 11825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "Pointer is not pointing to the start of a span"); 11835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 11845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ValidateAllocatedRegion(ptr, cl); 11855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (cl != 0) { 11875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ASSERT(!Static::pageheap()->GetDescriptor(p)->sample); 11885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ThreadCache* heap = GetCacheIfPresent(); 11895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (heap != NULL) { 11905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) heap->Deallocate(ptr, cl); 11915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } else { 11925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Delete directly into central cache 11935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tcmalloc::FL_Init(ptr); 11945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Static::central_cache()[cl].InsertRange(ptr, ptr, 1); 11955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 11965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } else { 11975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SpinLockHolder h(Static::pageheap_lock()); 11985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ASSERT(reinterpret_cast<uintptr_t>(ptr) % kPageSize == 0); 11995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ASSERT(span != NULL && span->start == p); 12005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (span->sample) { 12015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) StackTrace* st = reinterpret_cast<StackTrace*>(span->objects); 12025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tcmalloc::DLL_Remove(span); 12035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Static::stacktrace_allocator()->Delete(st); 12045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) span->objects = NULL; 12055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 12065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Static::pageheap()->Delete(span); 12075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 12085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 12095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// The default "do_free" that uses the default callback. 12115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void do_free(void* ptr) { 12125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return do_free_with_callback(ptr, &InvalidFree); 12135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 12145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// NOTE: some logic here is duplicated in GetOwnership (above), for 12165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// speed. If you change this function, look at that one too. 12175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline size_t GetSizeWithCallback(const void* ptr, 12185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t (*invalid_getsize_fn)(const void*)) { 12195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (ptr == NULL) 12205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 0; 12215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; 12225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t cl = Static::pageheap()->GetSizeClassIfCached(p); 12235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (cl != 0) { 12245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return Static::sizemap()->ByteSizeForClass(cl); 12255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } else { 12265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const Span *span = Static::pageheap()->GetDescriptor(p); 12275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (span == NULL) { // means we do not own this memory 12285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (*invalid_getsize_fn)(ptr); 12295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } else if (span->sizeclass != 0) { 12305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Static::pageheap()->CacheSizeClass(p, span->sizeclass); 12315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return Static::sizemap()->ByteSizeForClass(span->sizeclass); 12325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } else { 12335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return span->length << kPageShift; 12345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 12355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 12365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 12375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// This lets you call back to a given function pointer if ptr is invalid. 12395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// It is used primarily by windows code which wants a specialized callback. 12405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void* do_realloc_with_callback( 12415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* old_ptr, size_t new_size, 12425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void (*invalid_free_fn)(void*), 12435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t (*invalid_get_size_fn)(const void*)) { 12445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AddRoomForMark(&new_size); 12455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Get the size of the old entry 12465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const size_t old_size = GetSizeWithCallback(old_ptr, invalid_get_size_fn); 12475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Reallocate if the new size is larger than the old size, 12495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // or if the new size is significantly smaller than the old size. 12505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // We do hysteresis to avoid resizing ping-pongs: 12515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // . If we need to grow, grow to max(new_size, old_size * 1.X) 12525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // . Don't shrink unless new_size < old_size * 0.Y 12535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // X and Y trade-off time for wasted space. For now we do 1.25 and 0.5. 12542a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) const size_t min_growth = min(old_size / 4, 12552a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) (std::numeric_limits<size_t>::max)() - old_size); // Avoid overflow. 12562a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) const size_t lower_bound_to_grow = old_size + min_growth; 12572a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) const size_t upper_bound_to_shrink = old_size / 2; 12585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if ((new_size > old_size) || (new_size < upper_bound_to_shrink)) { 12595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Need to reallocate. 12605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* new_ptr = NULL; 12615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (new_size > old_size && new_size < lower_bound_to_grow) { 12635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_ptr = do_malloc_or_cpp_alloc(lower_bound_to_grow); 12645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 12655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ExcludeMarkFromSize(&new_size); // do_malloc will add space if needed. 12665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (new_ptr == NULL) { 12675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Either new_size is not a tiny increment, or last do_malloc failed. 12685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_ptr = do_malloc_or_cpp_alloc(new_size); 12695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 12705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (new_ptr == NULL) { 12715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return NULL; 12725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 12735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocHook::InvokeNewHook(new_ptr, new_size); 12745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memcpy(new_ptr, old_ptr, ((old_size < new_size) ? old_size : new_size)); 12755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocHook::InvokeDeleteHook(old_ptr); 12765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // We could use a variant of do_free() that leverages the fact 12775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // that we already know the sizeclass of old_ptr. The benefit 12785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // would be small, so don't bother. 12795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) do_free_with_callback(old_ptr, invalid_free_fn); 12805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return new_ptr; 12815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } else { 12825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // We still need to call hooks to report the updated size: 12835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocHook::InvokeDeleteHook(old_ptr); 12845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ExcludeMarkFromSize(&new_size); 12855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocHook::InvokeNewHook(old_ptr, new_size); 12865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return old_ptr; 12875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 12885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 12895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void* do_realloc(void* old_ptr, size_t new_size) { 12915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return do_realloc_with_callback(old_ptr, new_size, 12925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) &InvalidFree, &InvalidGetSizeForRealloc); 12935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 12945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// For use by exported routines below that want specific alignments 12965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 12975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Note: this code can be slow for alignments > 16, and can 12985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// significantly fragment memory. The expectation is that 12995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// memalign/posix_memalign/valloc/pvalloc will not be invoked very 13005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// often. This requirement simplifies our implementation and allows 13015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// us to tune for expected allocation patterns. 13025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void* do_memalign(size_t align, size_t size) { 13035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ASSERT((align & (align - 1)) == 0); 13045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ASSERT(align > 0); 13055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Marked in CheckMallocResult(), which is also inside SpanToMallocResult(). 13065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AddRoomForMark(&size); 13075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (size + align < size) return NULL; // Overflow 13085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Fall back to malloc if we would already align this memory access properly. 13105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (align <= AlignmentForSize(size)) { 13115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* p = do_malloc(size); 13125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ASSERT((reinterpret_cast<uintptr_t>(p) % align) == 0); 13135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return p; 13145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 13155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (Static::pageheap() == NULL) ThreadCache::InitModule(); 13175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Allocate at least one byte to avoid boundary conditions below 13195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (size == 0) size = 1; 13205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (size <= kMaxSize && align < kPageSize) { 13225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Search through acceptable size classes looking for one with 13235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // enough alignment. This depends on the fact that 13245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // InitSizeClasses() currently produces several size classes that 13255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // are aligned at powers of two. We will waste time and space if 13265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // we miss in the size class array, but that is deemed acceptable 13275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // since memalign() should be used rarely. 13285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int cl = Static::sizemap()->SizeClass(size); 13295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while (cl < kNumClasses && 13305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ((Static::sizemap()->class_to_size(cl) & (align - 1)) != 0)) { 13315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cl++; 13325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 13335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (cl < kNumClasses) { 13345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ThreadCache* heap = ThreadCache::GetCache(); 13355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size = Static::sizemap()->class_to_size(cl); 13365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return CheckMallocResult(heap->Allocate(size, cl)); 13375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 13385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 13395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // We will allocate directly from the page heap 13415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SpinLockHolder h(Static::pageheap_lock()); 13425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (align <= kPageSize) { 13445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Any page-level allocation will be fine 13455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // TODO: We could put the rest of this page in the appropriate 13465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // TODO: cache but it does not seem worth it. 13475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Span* span = Static::pageheap()->New(tcmalloc::pages(size)); 13485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return span == NULL ? NULL : SpanToMallocResult(span); 13495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 13505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Allocate extra pages and carve off an aligned portion 13525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const Length alloc = tcmalloc::pages(size + align); 13535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Span* span = Static::pageheap()->New(alloc); 13545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (span == NULL) return NULL; 13555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Skip starting portion so that we end up aligned 13575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Length skip = 0; 13585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while ((((span->start+skip) << kPageShift) & (align - 1)) != 0) { 13595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) skip++; 13605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 13615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ASSERT(skip < alloc); 13625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (skip > 0) { 13635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Span* rest = Static::pageheap()->Split(span, skip); 13645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Static::pageheap()->Delete(span); 13655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) span = rest; 13665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 13675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Skip trailing portion that we do not need to return 13695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const Length needed = tcmalloc::pages(size); 13705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ASSERT(span->length >= needed); 13715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (span->length > needed) { 13725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Span* trailer = Static::pageheap()->Split(span, needed); 13735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Static::pageheap()->Delete(trailer); 13745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 13755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SpanToMallocResult(span); 13765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 13775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Helpers for use by exported routines below: 13795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void do_malloc_stats() { 13815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PrintStats(1); 13825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 13835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline int do_mallopt(int cmd, int value) { 13855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 1; // Indicates error 13865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 13875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifdef HAVE_STRUCT_MALLINFO 13895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline struct mallinfo do_mallinfo() { 13905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TCMallocStats stats; 13915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ExtractStats(&stats, NULL, NULL, NULL); 13925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Just some of the fields are filled in. 13945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) struct mallinfo info; 13955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memset(&info, 0, sizeof(info)); 13965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Unfortunately, the struct contains "int" field, so some of the 13985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // size values will be truncated. 13995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) info.arena = static_cast<int>(stats.pageheap.system_bytes); 14005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) info.fsmblks = static_cast<int>(stats.thread_bytes 14015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) + stats.central_bytes 14025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) + stats.transfer_bytes); 14035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) info.fordblks = static_cast<int>(stats.pageheap.free_bytes + 14045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) stats.pageheap.unmapped_bytes); 14055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) info.uordblks = static_cast<int>(stats.pageheap.system_bytes 14065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) - stats.thread_bytes 14075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) - stats.central_bytes 14085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) - stats.transfer_bytes 14095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) - stats.pageheap.free_bytes 14105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) - stats.pageheap.unmapped_bytes); 14115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return info; 14135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 14145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif // HAVE_STRUCT_MALLINFO 14155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static SpinLock set_new_handler_lock(SpinLock::LINKER_INITIALIZED); 14175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void* cpp_alloc(size_t size, bool nothrow) { 14195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for (;;) { 14205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* p = do_malloc(size); 14215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifdef PREANSINEW 14225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return p; 14235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#else 14245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (p == NULL) { // allocation failed 14255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Get the current new handler. NB: this function is not 14265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // thread-safe. We make a feeble stab at making it so here, but 14275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // this lock only protects against tcmalloc interfering with 14285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // itself, not with other libraries calling set_new_handler. 14295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) std::new_handler nh; 14305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) { 14315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SpinLockHolder h(&set_new_handler_lock); 14325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nh = std::set_new_handler(0); 14335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (void) std::set_new_handler(nh); 14345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 14355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if (defined(__GNUC__) && !defined(__EXCEPTIONS)) || (defined(_HAS_EXCEPTIONS) && !_HAS_EXCEPTIONS) 14365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (nh) { 14375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Since exceptions are disabled, we don't really know if new_handler 14385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // failed. Assume it will abort if it fails. 14395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (*nh)(); 14405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) continue; 14415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 14425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 0; 14435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#else 14445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // If no new_handler is established, the allocation failed. 14455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (!nh) { 14465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (nothrow) return 0; 14475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) throw std::bad_alloc(); 14485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 14495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Otherwise, try the new_handler. If it returns, retry the 14505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // allocation. If it throws std::bad_alloc, fail the allocation. 14515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // if it throws something else, don't interfere. 14525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try { 14535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (*nh)(); 14545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } catch (const std::bad_alloc&) { 14555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (!nothrow) throw; 14565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return p; 14575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 14585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif // (defined(__GNUC__) && !defined(__EXCEPTIONS)) || (defined(_HAS_EXCEPTIONS) && !_HAS_EXCEPTIONS) 14595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } else { // allocation success 14605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return p; 14615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 14625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif // PREANSINEW 14635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 14645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 14655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void* cpp_memalign(size_t align, size_t size) { 14675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for (;;) { 14685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* p = do_memalign(align, size); 14695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifdef PREANSINEW 14705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return p; 14715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#else 14725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (p == NULL) { // allocation failed 14735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Get the current new handler. NB: this function is not 14745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // thread-safe. We make a feeble stab at making it so here, but 14755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // this lock only protects against tcmalloc interfering with 14765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // itself, not with other libraries calling set_new_handler. 14775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) std::new_handler nh; 14785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) { 14795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SpinLockHolder h(&set_new_handler_lock); 14805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nh = std::set_new_handler(0); 14815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (void) std::set_new_handler(nh); 14825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 14835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if (defined(__GNUC__) && !defined(__EXCEPTIONS)) || (defined(_HAS_EXCEPTIONS) && !_HAS_EXCEPTIONS) 14845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (nh) { 14855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Since exceptions are disabled, we don't really know if new_handler 14865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // failed. Assume it will abort if it fails. 14875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (*nh)(); 14885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) continue; 14895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 14905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 0; 14915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#else 14925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // If no new_handler is established, the allocation failed. 14935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (!nh) 14945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 0; 14955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Otherwise, try the new_handler. If it returns, retry the 14975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // allocation. If it throws std::bad_alloc, fail the allocation. 14985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // if it throws something else, don't interfere. 14995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try { 15005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (*nh)(); 15015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } catch (const std::bad_alloc&) { 15025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return p; 15035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 15045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif // (defined(__GNUC__) && !defined(__EXCEPTIONS)) || (defined(_HAS_EXCEPTIONS) && !_HAS_EXCEPTIONS) 15055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } else { // allocation success 15065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return p; 15075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 15085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif // PREANSINEW 15095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 15105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 15115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 15125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} // end unnamed namespace 15135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 15145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// As promised, the definition of this function, declared above. 15155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)size_t TCMallocImplementation::GetAllocatedSize(const void* ptr) { 15165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Chromium workaround for third-party code calling tc_malloc_size(NULL), see 15175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // http://code.google.com/p/chromium/issues/detail?id=118087 15185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Note: this is consistent with GLIBC's implementation of 15195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // malloc_usable_size(NULL). 15205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (ptr == NULL) 15215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 0; 15225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ASSERT(TCMallocImplementation::GetOwnership(ptr) 15235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) != TCMallocImplementation::kNotOwned); 15245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ExcludeSpaceForMark( 15255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) GetSizeWithCallback(ptr, &InvalidGetAllocatedSize)); 15265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 15275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 15285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void TCMallocImplementation::MarkThreadBusy() { 15295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Allocate to force the creation of a thread cache, but avoid 15305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // invoking any hooks. 15315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) do_free(do_malloc(0)); 15325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 15335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 15345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)//------------------------------------------------------------------- 15355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Exported routines 15365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)//------------------------------------------------------------------- 15375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 15385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" PERFTOOLS_DLL_DECL const char* tc_version( 15395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int* major, int* minor, const char** patch) __THROW { 15405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (major) *major = TC_VERSION_MAJOR; 15415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (minor) *minor = TC_VERSION_MINOR; 15425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (patch) *patch = TC_VERSION_PATCH; 15435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return TC_VERSION_STRING; 15445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 15455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 15465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// This function behaves similarly to MSVC's _set_new_mode. 15475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// If flag is 0 (default), calls to malloc will behave normally. 15485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// If flag is 1, calls to malloc will behave like calls to new, 15495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// and the std_new_handler will be invoked on failure. 15505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Returns the previous mode. 15515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" PERFTOOLS_DLL_DECL int tc_set_new_mode(int flag) __THROW { 15525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int old_mode = tc_new_mode; 15535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tc_new_mode = flag; 15545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return old_mode; 15555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 15565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 15575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifndef TCMALLOC_USING_DEBUGALLOCATION // debugallocation.cc defines its own 15585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 15595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// CAVEAT: The code structure below ensures that MallocHook methods are always 15605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// called from the stack frame of the invoked allocation function. 15615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// heap-checker.cc depends on this to start a stack trace from 15625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// the call to the (de)allocation function. 15635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 15645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" PERFTOOLS_DLL_DECL void* tc_malloc(size_t size) __THROW { 15655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* result = do_malloc_or_cpp_alloc(size); 15665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocHook::InvokeNewHook(result, size); 15675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return result; 15685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 15695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 15705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" PERFTOOLS_DLL_DECL void tc_free(void* ptr) __THROW { 15715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocHook::InvokeDeleteHook(ptr); 15725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) do_free(ptr); 15735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 15745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 15755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" PERFTOOLS_DLL_DECL void* tc_calloc(size_t n, 15765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t elem_size) __THROW { 15775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* result = do_calloc(n, elem_size); 15785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocHook::InvokeNewHook(result, n * elem_size); 15795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return result; 15805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 15815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 15825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" PERFTOOLS_DLL_DECL void tc_cfree(void* ptr) __THROW { 15835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocHook::InvokeDeleteHook(ptr); 15845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) do_free(ptr); 15855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 15865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 15875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" PERFTOOLS_DLL_DECL void* tc_realloc(void* old_ptr, 15885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t new_size) __THROW { 15895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (old_ptr == NULL) { 15905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* result = do_malloc_or_cpp_alloc(new_size); 15915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocHook::InvokeNewHook(result, new_size); 15925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return result; 15935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 15945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (new_size == 0) { 15955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocHook::InvokeDeleteHook(old_ptr); 15965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) do_free(old_ptr); 15975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return NULL; 15985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 15995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return do_realloc(old_ptr, new_size); 16005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 16015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 16025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" PERFTOOLS_DLL_DECL void* tc_new(size_t size) { 16035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* p = cpp_alloc(size, false); 16045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // We keep this next instruction out of cpp_alloc for a reason: when 16055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // it's in, and new just calls cpp_alloc, the optimizer may fold the 16065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // new call into cpp_alloc, which messes up our whole section-based 16075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // stacktracing (see ATTRIBUTE_SECTION, above). This ensures cpp_alloc 16085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // isn't the last thing this fn calls, and prevents the folding. 16095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocHook::InvokeNewHook(p, size); 16105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return p; 16115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 16125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 16135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" PERFTOOLS_DLL_DECL void* tc_new_nothrow(size_t size, const std::nothrow_t&) __THROW { 16145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* p = cpp_alloc(size, true); 16155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocHook::InvokeNewHook(p, size); 16165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return p; 16175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 16185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 16195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" PERFTOOLS_DLL_DECL void tc_delete(void* p) __THROW { 16205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocHook::InvokeDeleteHook(p); 16215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) do_free(p); 16225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 16235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 16245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Standard C++ library implementations define and use this 16255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// (via ::operator delete(ptr, nothrow)). 16265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// But it's really the same as normal delete, so we just do the same thing. 16275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" PERFTOOLS_DLL_DECL void tc_delete_nothrow(void* p, const std::nothrow_t&) __THROW { 16285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocHook::InvokeDeleteHook(p); 16295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) do_free(p); 16305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 16315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 16325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" PERFTOOLS_DLL_DECL void* tc_newarray(size_t size) { 16335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* p = cpp_alloc(size, false); 16345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // We keep this next instruction out of cpp_alloc for a reason: when 16355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // it's in, and new just calls cpp_alloc, the optimizer may fold the 16365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // new call into cpp_alloc, which messes up our whole section-based 16375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // stacktracing (see ATTRIBUTE_SECTION, above). This ensures cpp_alloc 16385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // isn't the last thing this fn calls, and prevents the folding. 16395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocHook::InvokeNewHook(p, size); 16405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return p; 16415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 16425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 16435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" PERFTOOLS_DLL_DECL void* tc_newarray_nothrow(size_t size, const std::nothrow_t&) 16445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) __THROW { 16455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* p = cpp_alloc(size, true); 16465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocHook::InvokeNewHook(p, size); 16475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return p; 16485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 16495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 16505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" PERFTOOLS_DLL_DECL void tc_deletearray(void* p) __THROW { 16515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocHook::InvokeDeleteHook(p); 16525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) do_free(p); 16535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 16545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 16555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_nothrow(void* p, const std::nothrow_t&) __THROW { 16565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocHook::InvokeDeleteHook(p); 16575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) do_free(p); 16585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 16595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 16605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" PERFTOOLS_DLL_DECL void* tc_memalign(size_t align, 16615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t size) __THROW { 16625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* result = do_memalign_or_cpp_memalign(align, size); 16635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocHook::InvokeNewHook(result, size); 16645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return result; 16655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 16665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 16675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" PERFTOOLS_DLL_DECL int tc_posix_memalign( 16685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void** result_ptr, size_t align, size_t size) __THROW { 16695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (((align % sizeof(void*)) != 0) || 16705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ((align & (align - 1)) != 0) || 16715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (align == 0)) { 16725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return EINVAL; 16735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 16745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 16755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* result = do_memalign_or_cpp_memalign(align, size); 16765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocHook::InvokeNewHook(result, size); 16775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (result == NULL) { 16785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ENOMEM; 16795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } else { 16805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *result_ptr = result; 16815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 0; 16825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 16835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 16845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 16855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static size_t pagesize = 0; 16865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 16875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" PERFTOOLS_DLL_DECL void* tc_valloc(size_t size) __THROW { 16885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Allocate page-aligned object of length >= size bytes 16895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (pagesize == 0) pagesize = getpagesize(); 16905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* result = do_memalign_or_cpp_memalign(pagesize, size); 16915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocHook::InvokeNewHook(result, size); 16925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return result; 16935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 16945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 16955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" PERFTOOLS_DLL_DECL void* tc_pvalloc(size_t size) __THROW { 16965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Round up size to a multiple of pagesize 16975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (pagesize == 0) pagesize = getpagesize(); 16985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (size == 0) { // pvalloc(0) should allocate one page, according to 16995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size = pagesize; // http://man.free4web.biz/man3/libmpatrol.3.html 17005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 17015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size = (size + pagesize - 1) & ~(pagesize - 1); 17025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* result = do_memalign_or_cpp_memalign(pagesize, size); 17035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MallocHook::InvokeNewHook(result, size); 17045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return result; 17055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 17065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" PERFTOOLS_DLL_DECL void tc_malloc_stats(void) __THROW { 17085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) do_malloc_stats(); 17095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 17105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" PERFTOOLS_DLL_DECL int tc_mallopt(int cmd, int value) __THROW { 17125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return do_mallopt(cmd, value); 17135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 17145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifdef HAVE_STRUCT_MALLINFO 17165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW { 17175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return do_mallinfo(); 17185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 17195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 17205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW { 17225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return MallocExtension::instance()->GetAllocatedSize(ptr); 17235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 17245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1725effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch#if defined(OS_LINUX) 1726effb81e5f8246d0db0270817048dc992db66e9fbBen Murdochextern "C" void* PERFTOOLS_DLL_DECL tc_malloc_skip_new_handler(size_t size) { 1727effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch void* result = do_malloc(size); 1728effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch MallocHook::InvokeNewHook(result, size); 1729effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch return result; 1730effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch} 1731effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch#endif 1732effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch 17335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif // TCMALLOC_USING_DEBUGALLOCATION 17345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1735effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch#if defined(OS_LINUX) 1736effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch// Alias the weak symbol in chromium to our implementation. 1737effb81e5f8246d0db0270817048dc992db66e9fbBen Murdochextern "C" __attribute__((visibility("default"), alias("tc_malloc_skip_new_handler"))) 1738effb81e5f8246d0db0270817048dc992db66e9fbBen Murdochvoid* tc_malloc_skip_new_handler_weak(size_t size); 1739effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch#endif 1740effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch 17415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// --- Validation implementation with an extra mark ---------------------------- 17425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// We will put a mark at the extreme end of each allocation block. We make 17435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// sure that we always allocate enough "extra memory" that we can fit in the 17445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// mark, and still provide the requested usable region. If ever that mark is 17455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// not as expected, then we know that the user is corrupting memory beyond their 17465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// request size, or that they have called free a second time without having 17475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// the memory allocated (again). This allows us to spot most double free()s, 17485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// but some can "slip by" or confuse our logic if the caller reallocates memory 17495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// (for a second use) before performing an evil double-free of a first 17505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// allocation 17515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// This code can be optimized, but for now, it is written to be most easily 17535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// understood, and flexible (since it is evolving a bit). Potential 17545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// optimizations include using other calculated data, such as class size, or 17555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// allocation size, which is known in the code above, but then is recalculated 17565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// below. Another potential optimization would be careful manual inlining of 17575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// code, but I *think* that the compile will probably do this for me, and I've 17585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// been careful to avoid aliasing issues that might make a compiler back-off. 17595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Evolution includes experimenting with different marks, to minimize the chance 17615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// that a mark would be misunderstood (missed corruption). The marks are meant 17625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// to be hashed encoding of the location, so that they can't be copied over a 17635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// different region (by accident) without being detected (most of the time). 17645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Enable the following define to turn on all the TCMalloc checking. 17665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// It will cost about 2% in performance, but it will catch double frees (most of 17675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// the time), and will often catch allocated-buffer overrun errors. This 17685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// validation is only active when TCMalloc is used as the allocator. 17695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifndef NDEBUG 17705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define TCMALLOC_VALIDATION 17715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 17725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if !defined(TCMALLOC_VALIDATION) 17745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static size_t ExcludeSpaceForMark(size_t size) { return size; } 17765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void AddRoomForMark(size_t* size) {} 17775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void ExcludeMarkFromSize(size_t* new_size) {} 17785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void MarkAllocatedRegion(void* ptr) {} 17795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void ValidateAllocatedRegion(void* ptr, size_t cl) {} 17805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#else // TCMALLOC_VALIDATION 17825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void DieFromDoubleFree() { 17845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Log(kCrash, __FILE__, __LINE__, "Attempt to double free"); 17855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 17865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void DieFromMemoryCorruption() { 17885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Log(kCrash, __FILE__, __LINE__, "Memory corrupted"); 17895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 17905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// We can either do byte marking, or whole word marking based on the following 17925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// define. char is as small as we can get, and word marking probably provides 17935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// more than enough bits that we won't miss a corruption. Any sized integral 17945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// type can be used, but we just define two examples. 17955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// #define TCMALLOC_SMALL_VALIDATION 17975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if defined (TCMALLOC_SMALL_VALIDATION) 17985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)typedef char MarkType; // char saves memory... int is more complete. 18005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static const MarkType kAllocationMarkMask = static_cast<MarkType>(0x36); 18015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 18025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#else 18035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 18045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)typedef int MarkType; // char saves memory... int is more complete. 18055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static const MarkType kAllocationMarkMask = static_cast<MarkType>(0xE1AB9536); 18065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 18075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 18085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 18095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// TODO(jar): See if use of reference rather than pointer gets better inlining, 18105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// or if macro is needed. My fear is that taking address map preclude register 18115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// allocation :-(. 18125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline static void AddRoomForMark(size_t* size) { 18135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *size += sizeof(kAllocationMarkMask); 18145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 18155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 18165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline static void ExcludeMarkFromSize(size_t* new_size) { 18175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *new_size -= sizeof(kAllocationMarkMask); 18185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 18195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 18205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline static size_t ExcludeSpaceForMark(size_t size) { 18215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return size - sizeof(kAllocationMarkMask); // Lie about size when asked. 18225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 18235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 18245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline static MarkType* GetMarkLocation(void* ptr) { 18255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t size = GetSizeWithCallback(ptr, &InvalidGetAllocatedSize); 18265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ASSERT(size % sizeof(kAllocationMarkMask) == 0); 18275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t last_index = (size / sizeof(kAllocationMarkMask)) - 1; 18285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return static_cast<MarkType*>(ptr) + last_index; 18295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 18305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 18315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// We hash in the mark location plus the pointer so that we effectively mix in 18325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// the size of the block. This means that if a span is used for different sizes 18335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// that the mark will be different. It would be good to hash in the size (which 18345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// we effectively get by using both mark location and pointer), but even better 18355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// would be to also include the class, as it concisely contains the entropy 18365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// found in the size (when we don't have large allocation), and there is less 18375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// risk of losing those bits to truncation. It would probably be good to combine 18385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// the high bits of size (capturing info about large blocks) with the class 18395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// (which is a 6 bit number). 18405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline static MarkType GetMarkValue(void* ptr, MarkType* mark) { 18415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* ptr2 = static_cast<void*>(mark); 18425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t offset1 = static_cast<char*>(ptr) - static_cast<char*>(NULL); 18435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t offset2 = static_cast<char*>(ptr2) - static_cast<char*>(NULL); 18445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static const int kInvariantBits = 2; 18455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ASSERT((offset1 >> kInvariantBits) << kInvariantBits == offset1); 18465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Note: low bits of both offsets are invariants due to alignment. High bits 18475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // of both offsets are the same (unless we have a large allocation). Avoid 18485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // XORing high bits together, as they will cancel for most small allocations. 18495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 18505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MarkType ret = kAllocationMarkMask; 18515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Using a little shift, we can safely XOR together both offsets. 18525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ret ^= static_cast<MarkType>(offset1 >> kInvariantBits) ^ 18535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static_cast<MarkType>(offset2); 18545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (sizeof(ret) == 1) { 18555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Try to bring some high level bits into the mix. 18565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ret += static_cast<MarkType>(offset1 >> 8) ^ 18575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static_cast<MarkType>(offset1 >> 16) ^ 18585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static_cast<MarkType>(offset1 >> 24) ; 18595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 18605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Hash in high bits on a 64 bit architecture. 18615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (sizeof(size_t) == 8 && sizeof(ret) == 4) 18625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ret += offset1 >> 16; 18635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (ret == 0) 18645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ret = kAllocationMarkMask; // Avoid common pattern of all zeros. 18655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ret; 18665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 18675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 18685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// TODO(jar): Use the passed in TCmalloc Class Index to calculate mark location 18695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// faster. The current implementation calls general functions, which have to 18705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// recalculate this in order to get the Class Size. This is a slow and wasteful 18715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// recomputation... but it is much more readable this way (for now). 18725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void ValidateAllocatedRegion(void* ptr, size_t cl) { 18735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (ptr == NULL) return; 18745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MarkType* mark = GetMarkLocation(ptr); 18755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MarkType allocated_mark = GetMarkValue(ptr, mark); 18765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MarkType current_mark = *mark; 18775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 18785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (current_mark == ~allocated_mark) 18795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) DieFromDoubleFree(); 18805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (current_mark != allocated_mark) 18815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) DieFromMemoryCorruption(); 18825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifndef NDEBUG 18835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // In debug mode, copy the mark into all the free'd region. 18845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t class_size = static_cast<size_t>(reinterpret_cast<char*>(mark) - 18855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reinterpret_cast<char*>(ptr)); 18865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memset(ptr, static_cast<char>(0x36), class_size); 18875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 18885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *mark = ~allocated_mark; // Distinctively not allocated. 18895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 18905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 18915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void MarkAllocatedRegion(void* ptr) { 18925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (ptr == NULL) return; 18935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MarkType* mark = GetMarkLocation(ptr); 18945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *mark = GetMarkValue(ptr, mark); 18955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 18965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 18975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif // TCMALLOC_VALIDATION 1898