15c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Copyright (c) 2005, 2007, Google Inc. 25c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// All rights reserved. 35c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Copyright (C) 2005, 2006, 2007, 2008, 2009, 2011 Apple Inc. All rights reserved. 402772c6a72f1ee0b226341a4f4439970c29fc861Ben Murdoch// 55c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Redistribution and use in source and binary forms, with or without 65c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// modification, are permitted provided that the following conditions are 75c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// met: 802772c6a72f1ee0b226341a4f4439970c29fc861Ben Murdoch// 95c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// * Redistributions of source code must retain the above copyright 105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// notice, this list of conditions and the following disclaimer. 115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// * Redistributions in binary form must reproduce the above 125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// copyright notice, this list of conditions and the following disclaimer 135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// in the documentation and/or other materials provided with the 145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// distribution. 155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// * Neither the name of Google Inc. nor the names of its 165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// contributors may be used to endorse or promote products derived from 175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// this software without specific prior written permission. 1802772c6a72f1ee0b226341a4f4439970c29fc861Ben Murdoch// 195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// --- 325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Author: Sanjay Ghemawat <opensource@google.com> 335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// A malloc that uses a per-thread cache to satisfy small malloc requests. 355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// (The time for malloc/free of a small object drops from 300 ns to 50 ns.) 365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// See doc/tcmalloc.html for a high-level 385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// description of how this malloc works. 395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// SYNCHRONIZATION 415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 1. The thread-specific lists are accessed without acquiring any locks. 425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// This is safe because each such list is only accessed by one thread. 435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 2. We have a lock per central free-list, and hold it while manipulating 445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// the central free list for a particular size. 455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 3. The central page allocator is protected by "pageheap_lock". 465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 4. The pagemap (which maps from page-number to descriptor), 475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// can be read without holding any locks, and written while holding 485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// the "pageheap_lock". 495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 5. To improve performance, a subset of the information one can get 505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// from the pagemap is cached in a data structure, pagemap_cache_, 515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// that atomically reads and writes its entries. This cache can be 525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// read and written without locking. 535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// This multi-threaded access to the pagemap is safe for fairly 555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// subtle reasons. We basically assume that when an object X is 565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// allocated by thread A and deallocated by thread B, there must 575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// have been appropriate synchronization in the handoff of object 585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// X from thread A to thread B. The same logic applies to pagemap_cache_. 595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// THE PAGEID-TO-SIZECLASS CACHE 615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Hot PageID-to-sizeclass mappings are held by pagemap_cache_. If this cache 625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// returns 0 for a particular PageID then that means "no information," not that 635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// the sizeclass is 0. The cache may have stale information for pages that do 645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// not hold the beginning of any free()'able object. Staleness is eliminated 655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// in Populate() for pages with sizeclass > 0 objects, and in do_malloc() and 665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// do_memalign() for all other relevant pages. 675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// TODO: Bias reclamation to larger addresses 695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// TODO: implement mallinfo/mallopt 705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// TODO: Better testing 715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 9/28/2003 (new page-level allocator replaces ptmalloc2): 735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// * malloc/free of small objects goes from ~300 ns to ~50 ns. 745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// * allocation of a reasonably complicated struct 755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// goes from about 1100 ns to about 300 ns. 765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include "config.h" 7893ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles)#include "wtf/FastMalloc.h" 795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 8093ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles)#include "wtf/Assertions.h" 8193ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles)#include "wtf/CPU.h" 8293ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles)#include "wtf/StdLibExtras.h" 8393ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles)#include "wtf/UnusedParam.h" 84926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 85591b958dee2cf159d33a0b931e6231072eaf38d5Ben Murdoch#if OS(DARWIN) 86591b958dee2cf159d33a0b931e6231072eaf38d5Ben Murdoch#include <AvailabilityMacros.h> 87591b958dee2cf159d33a0b931e6231072eaf38d5Ben Murdoch#endif 88591b958dee2cf159d33a0b931e6231072eaf38d5Ben Murdoch 895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include <limits> 905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if OS(WINDOWS) 915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include <windows.h> 925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#else 935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include <pthread.h> 945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 95f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles)#include <stdlib.h> 965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include <string.h> 975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#ifndef NO_TCMALLOC_SAMPLES 995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#define NO_TCMALLOC_SAMPLES 1005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 1015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 10253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)#if !USE(SYSTEM_MALLOC) && defined(NDEBUG) 1035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#define FORCE_SYSTEM_MALLOC 0 1045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#else 1055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#define FORCE_SYSTEM_MALLOC 1 1065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 1075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 108926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)// Harden the pointers stored in the TCMalloc linked lists 10953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)#if COMPILER(GCC) 110926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#define ENABLE_TCMALLOC_HARDENING 1 111926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#endif 112926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Use a background thread to periodically scavenge memory to release back to the system 1145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#define USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 1 1155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#ifndef NDEBUG 1175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)namespace WTF { 1185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if OS(WINDOWS) 1205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// TLS_OUT_OF_INDEXES is not defined on WinCE. 1225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#ifndef TLS_OUT_OF_INDEXES 1235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#define TLS_OUT_OF_INDEXES 0xffffffff 1245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 1255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static DWORD isForibiddenTlsIndex = TLS_OUT_OF_INDEXES; 1275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const LPVOID kTlsAllowValue = reinterpret_cast<LPVOID>(0); // Must be zero. 1285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const LPVOID kTlsForbiddenValue = reinterpret_cast<LPVOID>(1); 1295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if !ASSERT_DISABLED 1315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static bool isForbidden() 1325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 1335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // By default, fastMalloc is allowed so we don't allocate the 1345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // tls index unless we're asked to make it forbidden. If TlsSetValue 1355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // has not been called on a thread, the value returned by TlsGetValue is 0. 1365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return (isForibiddenTlsIndex != TLS_OUT_OF_INDEXES) && (TlsGetValue(isForibiddenTlsIndex) == kTlsForbiddenValue); 1375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 1385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 1395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void fastMallocForbid() 1415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 1425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (isForibiddenTlsIndex == TLS_OUT_OF_INDEXES) 1435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) isForibiddenTlsIndex = TlsAlloc(); // a little racey, but close enough for debug only 1445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TlsSetValue(isForibiddenTlsIndex, kTlsForbiddenValue); 1455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 1465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void fastMallocAllow() 1485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 1495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (isForibiddenTlsIndex == TLS_OUT_OF_INDEXES) 1505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return; 1515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TlsSetValue(isForibiddenTlsIndex, kTlsAllowValue); 1525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 1535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#else // !OS(WINDOWS) 1555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static pthread_key_t isForbiddenKey; 1575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static pthread_once_t isForbiddenKeyOnce = PTHREAD_ONCE_INIT; 1585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static void initializeIsForbiddenKey() 1595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 1605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_key_create(&isForbiddenKey, 0); 1615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 1625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if !ASSERT_DISABLED 1645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static bool isForbidden() 1655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 1665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_once(&isForbiddenKeyOnce, initializeIsForbiddenKey); 1675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return !!pthread_getspecific(isForbiddenKey); 1685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 1695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 1705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void fastMallocForbid() 1725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 1735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_once(&isForbiddenKeyOnce, initializeIsForbiddenKey); 1745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_setspecific(isForbiddenKey, &isForbiddenKey); 1755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 1765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void fastMallocAllow() 1785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 1795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_once(&isForbiddenKeyOnce, initializeIsForbiddenKey); 1805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_setspecific(isForbiddenKey, 0); 1815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 1825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif // OS(WINDOWS) 1835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} // namespace WTF 1855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif // NDEBUG 1865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)namespace WTF { 1885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 18902772c6a72f1ee0b226341a4f4439970c29fc861Ben Murdochvoid* fastZeroedMalloc(size_t n) 1905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 1915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void* result = fastMalloc(n); 1925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) memset(result, 0, n); 1935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return result; 1945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 1955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)char* fastStrDup(const char* src) 1975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 1985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t len = strlen(src) + 1; 1995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) char* dup = static_cast<char*>(fastMalloc(len)); 2005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) memcpy(dup, src, len); 2015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return dup; 2025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 2035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} // namespace WTF 2055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if FORCE_SYSTEM_MALLOC 2075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if OS(DARWIN) 2095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include <malloc/malloc.h> 2105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#elif OS(WINDOWS) 2115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include <malloc.h> 2125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 2135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)namespace WTF { 2155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 216926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)size_t fastMallocGoodSize(size_t bytes) 217926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles){ 218926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#if OS(DARWIN) 219926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return malloc_good_size(bytes); 220926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#else 221926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return bytes; 222926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#endif 223926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)} 224926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 22502772c6a72f1ee0b226341a4f4439970c29fc861Ben Murdochvoid* fastMalloc(size_t n) 2265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 2275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(!isForbidden()); 2285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void* result = malloc(n); 23053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) ASSERT(result); // We expect tcmalloc underneath, which would crash instead of getting here. 2315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return result; 2335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 2345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void* fastCalloc(size_t n_elements, size_t element_size) 2365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 2375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(!isForbidden()); 2385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void* result = calloc(n_elements, element_size); 24053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) ASSERT(result); // We expect tcmalloc underneath, which would crash instead of getting here. 2415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return result; 2435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 2445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void fastFree(void* p) 2465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 2475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(!isForbidden()); 2485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) free(p); 2505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 2515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void* fastRealloc(void* p, size_t n) 2535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 2545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(!isForbidden()); 2555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void* result = realloc(p, n); 25753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) ASSERT(result); // We expect tcmalloc underneath, which would crash instead of getting here. 25853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 2595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return result; 2605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 2615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void releaseFastMallocFreeMemory() { } 26302772c6a72f1ee0b226341a4f4439970c29fc861Ben Murdoch 2645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)FastMallocStatistics fastMallocStatistics() 2655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 2665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) FastMallocStatistics statistics = { 0, 0, 0 }; 2675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return statistics; 2685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 2695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} // namespace WTF 2715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if OS(DARWIN) 2735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// This symbol is present in the JavaScriptCore exports file even when FastMalloc is disabled. 2745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// It will never be used in this case, so it's type and value are less interesting than its presence. 27553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)extern "C" const int jscore_fastmalloc_introspection = 0; 2765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 2775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#else // FORCE_SYSTEM_MALLOC 2795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 28053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)#include "Compiler.h" 2815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include "TCPackedCache.h" 2825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include "TCPageMap.h" 2835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include "TCSpinLock.h" 2845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include "TCSystemAlloc.h" 2855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include <algorithm> 286f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles)#include <errno.h> 2875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include <pthread.h> 2885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include <stdarg.h> 2895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include <stddef.h> 2905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include <stdio.h> 2915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if OS(UNIX) 2925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include <unistd.h> 2935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 2945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if OS(WINDOWS) 2955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#ifndef WIN32_LEAN_AND_MEAN 2965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#define WIN32_LEAN_AND_MEAN 2975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 2985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include <windows.h> 2995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 3005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if OS(DARWIN) 3025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include "MallocZoneSupport.h" 303591b958dee2cf159d33a0b931e6231072eaf38d5Ben Murdoch#include "wtf/HashSet.h" 304591b958dee2cf159d33a0b931e6231072eaf38d5Ben Murdoch#include "wtf/Vector.h" 305f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles)#else 306f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles)#include "wtf/CurrentTime.h" 3075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 3085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if HAVE(DISPATCH_H) 3105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include <dispatch/dispatch.h> 3115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 3125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 313926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#ifdef __has_include 314926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#if __has_include(<System/pthread_machdep.h>) 315926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 3165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include <System/pthread_machdep.h> 3175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 318926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#endif 3195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 3205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#ifndef PRIuS 3225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#define PRIuS "zu" 3235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 3245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Calling pthread_getspecific through a global function pointer is faster than a normal 3265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// call to the function on Mac OS X, and it's used in performance-critical code. So we 3275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// use a function pointer. But that's not necessarily faster on other platforms, and we had 3285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// problems with this technique on Windows, so we'll do this only on Mac OS X. 3295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if OS(DARWIN) 3305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if !USE(PTHREAD_GETSPECIFIC_DIRECT) 3315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static void* (*pthread_getspecific_function_pointer)(pthread_key_t) = pthread_getspecific; 3325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#define pthread_getspecific(key) pthread_getspecific_function_pointer(key) 3335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#else 3345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#define pthread_getspecific(key) _pthread_getspecific_direct(key) 3355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#define pthread_setspecific(key, val) _pthread_setspecific_direct(key, (val)) 3365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 3375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 3385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#define DEFINE_VARIABLE(type, name, value, meaning) \ 3405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) namespace FLAG__namespace_do_not_use_directly_use_DECLARE_##type##_instead { \ 3415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) type FLAGS_##name(value); \ 3425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) char FLAGS_no##name; \ 3435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } \ 3445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) using FLAG__namespace_do_not_use_directly_use_DECLARE_##type##_instead::FLAGS_##name 34502772c6a72f1ee0b226341a4f4439970c29fc861Ben Murdoch 3465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#define DEFINE_int64(name, value, meaning) \ 3475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) DEFINE_VARIABLE(int64_t, name, value, meaning) 34802772c6a72f1ee0b226341a4f4439970c29fc861Ben Murdoch 3495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#define DEFINE_double(name, value, meaning) \ 3505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) DEFINE_VARIABLE(double, name, value, meaning) 3515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)namespace WTF { 3535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#define malloc fastMalloc 3555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#define calloc fastCalloc 3565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#define free fastFree 3575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#define realloc fastRealloc 3585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#define MESSAGE LOG_ERROR 3605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#define CHECK_CONDITION ASSERT 3615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 362926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)static const char kLLHardeningMask = 0; 363926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)template <unsigned> struct EntropySource; 364926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)template <> struct EntropySource<4> { 365926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static uint32_t value() 366926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) { 3675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if OS(DARWIN) 368926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return arc4random(); 369926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#else 370926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return static_cast<uint32_t>(static_cast<uintptr_t>(currentTime() * 10000) ^ reinterpret_cast<uintptr_t>(&kLLHardeningMask)); 371926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#endif 372926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) } 373926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)}; 3745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 375926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)template <> struct EntropySource<8> { 376926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static uint64_t value() 377926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) { 378926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return EntropySource<4>::value() | (static_cast<uint64_t>(EntropySource<4>::value()) << 32); 379926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) } 380926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)}; 3815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 382926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#if ENABLE(TCMALLOC_HARDENING) 383926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)/* 384926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) * To make it harder to exploit use-after free style exploits 385926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) * we mask the addresses we put into our linked lists with the 386926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) * address of kLLHardeningMask. Due to ASLR the address of 387926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) * kLLHardeningMask should be sufficiently randomized to make direct 388926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) * freelist manipulation much more difficult. 389926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) */ 390926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)enum { 391926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) MaskKeyShift = 13 392926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)}; 3935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 39402772c6a72f1ee0b226341a4f4439970c29fc861Ben Murdochstatic ALWAYS_INLINE uintptr_t internalEntropyValue() 395926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles){ 396926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static uintptr_t value = EntropySource<sizeof(uintptr_t)>::value() | 1; 397926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ASSERT(value); 398926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return value; 399926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)} 4005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 401926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#define HARDENING_ENTROPY internalEntropyValue() 402926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#define ROTATE_VALUE(value, amount) (((value) >> (amount)) | ((value) << (sizeof(value) * 8 - (amount)))) 403926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#define XOR_MASK_PTR_WITH_KEY(ptr, key, entropy) (reinterpret_cast<typeof(ptr)>(reinterpret_cast<uintptr_t>(ptr)^(ROTATE_VALUE(reinterpret_cast<uintptr_t>(key), MaskKeyShift)^entropy))) 4045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 406926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)static ALWAYS_INLINE uint32_t freedObjectStartPoison() 407926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles){ 408926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static uint32_t value = EntropySource<sizeof(uint32_t)>::value() | 1; 409926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ASSERT(value); 410926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return value; 411926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)} 412926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 413926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)static ALWAYS_INLINE uint32_t freedObjectEndPoison() 414926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles){ 415926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static uint32_t value = EntropySource<sizeof(uint32_t)>::value() | 1; 416926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ASSERT(value); 417926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return value; 418926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)} 419926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 420926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#define PTR_TO_UINT32(ptr) static_cast<uint32_t>(reinterpret_cast<uintptr_t>(ptr)) 421926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#define END_POISON_INDEX(allocationSize) (((allocationSize) - sizeof(uint32_t)) / sizeof(uint32_t)) 422926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#define POISON_ALLOCATION(allocation, allocationSize) do { \ 423926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ASSERT((allocationSize) >= 2 * sizeof(uint32_t)); \ 424926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) reinterpret_cast<uint32_t*>(allocation)[0] = 0xbadbeef1; \ 425926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) reinterpret_cast<uint32_t*>(allocation)[1] = 0xbadbeef3; \ 426926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if ((allocationSize) < 4 * sizeof(uint32_t)) \ 427926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) break; \ 428926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) reinterpret_cast<uint32_t*>(allocation)[2] = 0xbadbeef5; \ 429926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) reinterpret_cast<uint32_t*>(allocation)[END_POISON_INDEX(allocationSize)] = 0xbadbeef7; \ 430926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)} while (false); 431926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 432926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#define POISON_DEALLOCATION_EXPLICIT(allocation, allocationSize, startPoison, endPoison) do { \ 433926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ASSERT((allocationSize) >= 2 * sizeof(uint32_t)); \ 434926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) reinterpret_cast<uint32_t*>(allocation)[0] = 0xbadbeef9; \ 435926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) reinterpret_cast<uint32_t*>(allocation)[1] = 0xbadbeefb; \ 436926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if ((allocationSize) < 4 * sizeof(uint32_t)) \ 437926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) break; \ 438926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) reinterpret_cast<uint32_t*>(allocation)[2] = (startPoison) ^ PTR_TO_UINT32(allocation); \ 439926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) reinterpret_cast<uint32_t*>(allocation)[END_POISON_INDEX(allocationSize)] = (endPoison) ^ PTR_TO_UINT32(allocation); \ 440926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)} while (false) 441926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 442926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#define POISON_DEALLOCATION(allocation, allocationSize) \ 443926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) POISON_DEALLOCATION_EXPLICIT(allocation, (allocationSize), freedObjectStartPoison(), freedObjectEndPoison()) 444926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 445926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#define MAY_BE_POISONED(allocation, allocationSize) (((allocationSize) >= 4 * sizeof(uint32_t)) && ( \ 446926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) (reinterpret_cast<uint32_t*>(allocation)[2] == (freedObjectStartPoison() ^ PTR_TO_UINT32(allocation))) || \ 447926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) (reinterpret_cast<uint32_t*>(allocation)[END_POISON_INDEX(allocationSize)] == (freedObjectEndPoison() ^ PTR_TO_UINT32(allocation))) \ 448926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles))) 449926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 450926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#define IS_DEFINITELY_POISONED(allocation, allocationSize) (((allocationSize) < 4 * sizeof(uint32_t)) || ( \ 451926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) (reinterpret_cast<uint32_t*>(allocation)[2] == (freedObjectStartPoison() ^ PTR_TO_UINT32(allocation))) && \ 452926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) (reinterpret_cast<uint32_t*>(allocation)[END_POISON_INDEX(allocationSize)] == (freedObjectEndPoison() ^ PTR_TO_UINT32(allocation))) \ 453926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles))) 4545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#else 4565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 457926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#define POISON_ALLOCATION(allocation, allocationSize) 458926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#define POISON_DEALLOCATION(allocation, allocationSize) 459926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#define POISON_DEALLOCATION_EXPLICIT(allocation, allocationSize, startPoison, endPoison) 460926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#define MAY_BE_POISONED(allocation, allocationSize) (false) 461926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#define IS_DEFINITELY_POISONED(allocation, allocationSize) (true) 462926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#define XOR_MASK_PTR_WITH_KEY(ptr, key, entropy) (((void)entropy), ((void)key), ptr) 463926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 464926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#define HARDENING_ENTROPY 0 4655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 4675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)//------------------------------------------------------------------- 4695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Configuration 4705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)//------------------------------------------------------------------- 4715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Not all possible combinations of the following parameters make 4735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// sense. In particular, if kMaxSize increases, you may have to 4745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// increase kNumClasses as well. 4755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const size_t kPageShift = 12; 4765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const size_t kPageSize = 1 << kPageShift; 4775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const size_t kMaxSize = 8u * kPageSize; 4785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const size_t kAlignShift = 3; 4795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const size_t kAlignment = 1 << kAlignShift; 4805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const size_t kNumClasses = 68; 4815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Allocates a big block of memory for the pagemap once we reach more than 4835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 128MB 4845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const size_t kPageMapBigAllocationThreshold = 128 << 20; 4855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Minimum number of pages to fetch from system at a time. Must be 4875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// significantly bigger than kPageSize to amortize system-call 4885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// overhead, and also to reduce external fragementation. Also, we 4895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// should keep this value big because various incarnations of Linux 4905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// have small limits on the number of mmap() regions per 4915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// address-space. 4925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const size_t kMinSystemAlloc = 1 << (20 - kPageShift); 4935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Number of objects to move between a per-thread list and a central 4955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// list in one shot. We want this to be not too small so we can 4965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// amortize the lock overhead for accessing the central list. Making 4975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// it too big may temporarily cause unnecessary memory wastage in the 4985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// per-thread free list until the scavenger cleans up the list. 4995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static int num_objects_to_move[kNumClasses]; 5005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Maximum length we allow a per-thread free-list to have before we 5025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// move objects from it into the corresponding central free-list. We 5035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// want this big to avoid locking the central free-list too often. It 5045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// should not hurt to make this list somewhat big because the 5055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// scavenging code will shrink it down when its contents are not in use. 5065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const int kMaxFreeListLength = 256; 5075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Lower and upper bounds on the per-thread cache sizes 5095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const size_t kMinThreadCacheSize = kMaxSize * 2; 5105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const size_t kMaxThreadCacheSize = 2 << 20; 5115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Default bound on the total amount of thread caches 5135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const size_t kDefaultOverallThreadCacheSize = 16 << 20; 5145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// For all span-lengths < kMaxPages we keep an exact-size list. 5165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// REQUIRED: kMaxPages >= kMinSystemAlloc; 5175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const size_t kMaxPages = kMinSystemAlloc; 5185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)/* The smallest prime > 2^n */ 5205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static int primes_list[] = { 5215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Small values might cause high rates of sampling 5225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // and hence commented out. 5235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // 2, 5, 11, 17, 37, 67, 131, 257, 5245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // 521, 1031, 2053, 4099, 8209, 16411, 5255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 32771, 65537, 131101, 262147, 524309, 1048583, 5265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2097169, 4194319, 8388617, 16777259, 33554467 }; 5275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Twice the approximate gap between sampling actions. 5295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// I.e., we take one sample approximately once every 5305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// tcmalloc_sample_parameter/2 5315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// bytes of allocation, i.e., ~ once every 128KB. 5325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Must be a prime number. 5335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#ifdef NO_TCMALLOC_SAMPLES 5345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)DEFINE_int64(tcmalloc_sample_parameter, 0, 5355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "Unused: code is compiled with NO_TCMALLOC_SAMPLES"); 5365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static size_t sample_period = 0; 5375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#else 5385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)DEFINE_int64(tcmalloc_sample_parameter, 262147, 5395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "Twice the approximate gap between sampling actions." 5405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) " Must be a prime number. Otherwise will be rounded up to a " 5415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) " larger prime number"); 5425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static size_t sample_period = 262147; 5435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 5445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Protects sample_period above 5465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static SpinLock sample_period_lock = SPINLOCK_INITIALIZER; 5475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Parameters for controlling how fast memory is returned to the OS. 5495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)DEFINE_double(tcmalloc_release_rate, 1, 5515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "Rate at which we release unused memory to the system. " 5525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "Zero means we never release memory back to the system. " 5535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "Increase this flag to return memory faster; decrease it " 5545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "to return memory slower. Reasonable rates are in the " 5555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "range [0,10]"); 5565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)//------------------------------------------------------------------- 5585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Mapping from size to size_class and vice versa 5595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)//------------------------------------------------------------------- 5605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Sizes <= 1024 have an alignment >= 8. So for such sizes we have an 5625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// array indexed by ceil(size/8). Sizes > 1024 have an alignment >= 128. 5635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// So for these larger sizes we have an array indexed by ceil(size/128). 5645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 5655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// We flatten both logical arrays into one physical array and use 5665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// arithmetic to compute an appropriate index. The constants used by 5675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// ClassIndex() were selected to make the flattening work. 5685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 5695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Examples: 5705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Size Expression Index 5715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// ------------------------------------------------------- 5725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 0 (0 + 7) / 8 0 5735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 1 (1 + 7) / 8 1 5745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// ... 5755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 1024 (1024 + 7) / 8 128 5765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 1025 (1025 + 127 + (120<<7)) / 128 129 5775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// ... 5785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 32768 (32768 + 127 + (120<<7)) / 128 376 5795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const size_t kMaxSmallSize = 1024; 5805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const int shift_amount[2] = { 3, 7 }; // For divides by 8 or 128 5815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const int add_amount[2] = { 7, 127 + (120 << 7) }; 5825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static unsigned char class_array[377]; 5835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Compute index of the class_array[] entry for a given size 5855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static inline int ClassIndex(size_t s) { 5865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const int i = (s > kMaxSmallSize); 5875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return static_cast<int>((s + add_amount[i]) >> shift_amount[i]); 5885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 5895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Mapping from size class to max size storable in that class 5915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static size_t class_to_size[kNumClasses]; 5925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Mapping from size class to number of pages to allocate at a time 5945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static size_t class_to_pages[kNumClasses]; 5955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 596926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)// Hardened singly linked list. We make this a class to allow compiler to 597926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)// statically prevent mismatching hardened and non-hardened list 598926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)class HardenedSLL { 599926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)public: 600926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static ALWAYS_INLINE HardenedSLL create(void* value) 601926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) { 602926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL result; 603926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) result.m_value = value; 604926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return result; 605926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) } 606926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 607926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static ALWAYS_INLINE HardenedSLL null() 608926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) { 609926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL result; 610926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) result.m_value = 0; 611926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return result; 612926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) } 613926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 614926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ALWAYS_INLINE void setValue(void* value) { m_value = value; } 615926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ALWAYS_INLINE void* value() const { return m_value; } 616926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ALWAYS_INLINE bool operator!() const { return !m_value; } 617926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) typedef void* (HardenedSLL::*UnspecifiedBoolType); 618926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ALWAYS_INLINE operator UnspecifiedBoolType() const { return m_value ? &HardenedSLL::m_value : 0; } 619926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 620926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) bool operator!=(const HardenedSLL& other) const { return m_value != other.m_value; } 621926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) bool operator==(const HardenedSLL& other) const { return m_value == other.m_value; } 622926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 623926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)private: 624926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) void* m_value; 625926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)}; 626926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 6275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// TransferCache is used to cache transfers of num_objects_to_move[size_class] 6285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// back and forth between thread caches and the central cache for a given size 6295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// class. 6305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)struct TCEntry { 631926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL head; // Head of chain of objects. 632926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL tail; // Tail of chain of objects. 6335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)}; 6345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// A central cache freelist can have anywhere from 0 to kNumTransferEntries 6355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// slots to put link list chains into. To keep memory usage bounded the total 6365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// number of TCEntries across size classes is fixed. Currently each size 6375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// class is initially given one TCEntry which also means that the maximum any 6385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// one class can have is kNumClasses. 6395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const int kNumTransferEntries = kNumClasses; 6405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 6415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Note: the following only works for "n"s that fit in 32-bits, but 6425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// that is fine since we only use it for small sizes. 6435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static inline int LgFloor(size_t n) { 6445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int log = 0; 6455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (int i = 4; i >= 0; --i) { 6465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int shift = (1 << i); 6475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t x = n >> shift; 6485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (x != 0) { 6495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) n = x; 6505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) log += shift; 6515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 6525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 6535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(n == 1); 6545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return log; 6555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 6565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 657926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)// Functions for using our simple hardened singly linked list 658926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)static ALWAYS_INLINE HardenedSLL SLL_Next(HardenedSLL t, uintptr_t entropy) { 659926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return HardenedSLL::create(XOR_MASK_PTR_WITH_KEY(*(reinterpret_cast<void**>(t.value())), t.value(), entropy)); 6605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 6615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 662926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)static ALWAYS_INLINE void SLL_SetNext(HardenedSLL t, HardenedSLL n, uintptr_t entropy) { 663926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) *(reinterpret_cast<void**>(t.value())) = XOR_MASK_PTR_WITH_KEY(n.value(), t.value(), entropy); 6645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 6655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 666926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)static ALWAYS_INLINE void SLL_Push(HardenedSLL* list, HardenedSLL element, uintptr_t entropy) { 667926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) SLL_SetNext(element, *list, entropy); 6685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) *list = element; 6695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 6705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 671926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)static ALWAYS_INLINE HardenedSLL SLL_Pop(HardenedSLL *list, uintptr_t entropy) { 672926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL result = *list; 673926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) *list = SLL_Next(*list, entropy); 6745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return result; 6755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 6765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 6775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Remove N elements from a linked list to which head points. head will be 6785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// modified to point to the new head. start and end will point to the first 6795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// and last nodes of the range. Note that end will point to NULL after this 6805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// function is called. 681926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 682926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)static ALWAYS_INLINE void SLL_PopRange(HardenedSLL* head, int N, HardenedSLL *start, HardenedSLL *end, uintptr_t entropy) { 6835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (N == 0) { 684926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) *start = HardenedSLL::null(); 685926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) *end = HardenedSLL::null(); 6865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return; 6875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 6885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 689926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL tmp = *head; 6905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (int i = 1; i < N; ++i) { 691926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) tmp = SLL_Next(tmp, entropy); 6925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 6935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 6945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) *start = *head; 6955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) *end = tmp; 696926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) *head = SLL_Next(tmp, entropy); 6975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Unlink range from list. 698926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) SLL_SetNext(tmp, HardenedSLL::null(), entropy); 6995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 7005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 701926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)static ALWAYS_INLINE void SLL_PushRange(HardenedSLL *head, HardenedSLL start, HardenedSLL end, uintptr_t entropy) { 7025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (!start) return; 703926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) SLL_SetNext(end, *head, entropy); 7045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) *head = start; 7055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 7065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 707926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)static ALWAYS_INLINE size_t SLL_Size(HardenedSLL head, uintptr_t entropy) { 7085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int count = 0; 7095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) while (head) { 7105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) count++; 711926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) head = SLL_Next(head, entropy); 7125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 7135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return count; 7145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 7155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 7165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Setup helper functions. 7175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 7185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static ALWAYS_INLINE size_t SizeClass(size_t size) { 7195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return class_array[ClassIndex(size)]; 7205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 7215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 7225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Get the byte-size for a specified class 7235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static ALWAYS_INLINE size_t ByteSizeForClass(size_t cl) { 7245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return class_to_size[cl]; 7255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 7265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static int NumMoveSize(size_t size) { 7275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (size == 0) return 0; 7285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Use approx 64k transfers between thread and central caches. 7295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int num = static_cast<int>(64.0 * 1024.0 / size); 7305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (num < 2) num = 2; 7315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Clamp well below kMaxFreeListLength to avoid ping pong between central 7325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // and thread caches. 7335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (num > static_cast<int>(0.8 * kMaxFreeListLength)) 7345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) num = static_cast<int>(0.8 * kMaxFreeListLength); 7355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 7365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Also, avoid bringing in too many objects into small object free 7375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // lists. There are lots of such lists, and if we allow each one to 7385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // fetch too many at a time, we end up having to scavenge too often 7395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // (especially when there are lots of threads and each thread gets a 7405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // small allowance for its thread cache). 7415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // 7425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // TODO: Make thread cache free list sizes dynamic so that we do not 7435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // have to equally divide a fixed resource amongst lots of threads. 7445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (num > 32) num = 32; 7455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 7465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return num; 7475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 7485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 7495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Initialize the mapping arrays 7505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static void InitSizeClasses() { 7515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Do some sanity checking on add_amount[]/shift_amount[]/class_array[] 7525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (ClassIndex(0) < 0) { 7535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) MESSAGE("Invalid class index %d for size 0\n", ClassIndex(0)); 7545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CRASH(); 7555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 7565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (static_cast<size_t>(ClassIndex(kMaxSize)) >= sizeof(class_array)) { 7575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) MESSAGE("Invalid class index %d for kMaxSize\n", ClassIndex(kMaxSize)); 7585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CRASH(); 7595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 7605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 7615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Compute the size classes we want to use 7625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t sc = 1; // Next size class to assign 7635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) unsigned char alignshift = kAlignShift; 7645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int last_lg = -1; 7655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (size_t size = kAlignment; size <= kMaxSize; size += (1 << alignshift)) { 7665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int lg = LgFloor(size); 7675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (lg > last_lg) { 7685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Increase alignment every so often. 7695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // 7705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Since we double the alignment every time size doubles and 7715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // size >= 128, this means that space wasted due to alignment is 7725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // at most 16/128 i.e., 12.5%. Plus we cap the alignment at 256 7735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // bytes, so the space wasted as a percentage starts falling for 7745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // sizes > 2K. 7755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if ((lg >= 7) && (alignshift < 8)) { 7765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) alignshift++; 7775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 7785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) last_lg = lg; 7795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 7805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 7815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Allocate enough pages so leftover is less than 1/8 of total. 7825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // This bounds wasted space to at most 12.5%. 7835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t psize = kPageSize; 7845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) while ((psize % size) > (psize >> 3)) { 7855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) psize += kPageSize; 7865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 7875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const size_t my_pages = psize >> kPageShift; 7885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 7895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (sc > 1 && my_pages == class_to_pages[sc-1]) { 7905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // See if we can merge this into the previous class without 7915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // increasing the fragmentation of the previous class. 7925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const size_t my_objects = (my_pages << kPageShift) / size; 7935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const size_t prev_objects = (class_to_pages[sc-1] << kPageShift) 7945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) / class_to_size[sc-1]; 7955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (my_objects == prev_objects) { 7965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Adjust last class to include this size 7975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) class_to_size[sc-1] = size; 7985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) continue; 7995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 8005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 8015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 8025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Add new class 8035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) class_to_pages[sc] = my_pages; 8045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) class_to_size[sc] = size; 8055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) sc++; 8065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 8075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (sc != kNumClasses) { 8085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) MESSAGE("wrong number of size classes: found %" PRIuS " instead of %d\n", 8095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) sc, int(kNumClasses)); 8105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CRASH(); 8115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 8125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 8135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Initialize the mapping arrays 8145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int next_size = 0; 8155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (unsigned char c = 1; c < kNumClasses; c++) { 8165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const size_t max_size_in_class = class_to_size[c]; 8175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (size_t s = next_size; s <= max_size_in_class; s += kAlignment) { 8185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) class_array[ClassIndex(s)] = c; 8195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 8205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) next_size = static_cast<int>(max_size_in_class + kAlignment); 8215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 8225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 8235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Double-check sizes just to be safe 8245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (size_t size = 0; size <= kMaxSize; size++) { 8255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const size_t sc = SizeClass(size); 8265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (sc == 0) { 8275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) MESSAGE("Bad size class %" PRIuS " for %" PRIuS "\n", sc, size); 8285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CRASH(); 8295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 8305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (sc > 1 && size <= class_to_size[sc-1]) { 8315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) MESSAGE("Allocating unnecessarily large class %" PRIuS " for %" PRIuS 8325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "\n", sc, size); 8335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CRASH(); 8345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 8355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (sc >= kNumClasses) { 8365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) MESSAGE("Bad size class %" PRIuS " for %" PRIuS "\n", sc, size); 8375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CRASH(); 8385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 8395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const size_t s = class_to_size[sc]; 8405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (size > s) { 8415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) MESSAGE("Bad size %" PRIuS " for %" PRIuS " (sc = %" PRIuS ")\n", s, size, sc); 8425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CRASH(); 8435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 8445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (s == 0) { 8455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) MESSAGE("Bad size %" PRIuS " for %" PRIuS " (sc = %" PRIuS ")\n", s, size, sc); 8465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CRASH(); 8475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 8485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 8495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 8505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Initialize the num_objects_to_move array. 8515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (size_t cl = 1; cl < kNumClasses; ++cl) { 8525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) num_objects_to_move[cl] = NumMoveSize(ByteSizeForClass(cl)); 8535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 8545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 8555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 8565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// ------------------------------------------------------------------------- 8575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Simple allocator for objects of a specified type. External locking 8585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// is required before accessing one of these objects. 8595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// ------------------------------------------------------------------------- 8605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 8615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Metadata allocator -- keeps stats about how many bytes allocated 8625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static uint64_t metadata_system_bytes = 0; 8635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static void* MetaDataAlloc(size_t bytes) { 8645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void* result = TCMalloc_SystemAlloc(bytes, 0); 8655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (result != NULL) { 8665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) metadata_system_bytes += bytes; 8675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 8685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return result; 8695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 8705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 8715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)template <class T> 8725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class PageHeapAllocator { 8735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) private: 8745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // How much to allocate from system at a time 8755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) static const size_t kAllocIncrement = 32 << 10; 8765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 8775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Aligned size of T 8785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) static const size_t kAlignedSize 8795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) = (((sizeof(T) + kAlignment - 1) / kAlignment) * kAlignment); 8805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 8815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Free area from which to carve new objects 8825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) char* free_area_; 8835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t free_avail_; 8845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 8855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Linked list of all regions allocated by this allocator 886926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL allocated_regions_; 8875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 8885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Free list of already carved objects 889926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL free_list_; 8905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 8915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Number of allocated but unfreed objects 8925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int inuse_; 893926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) uintptr_t entropy_; 8945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 8955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) public: 896926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) void Init(uintptr_t entropy) { 8975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(kAlignedSize <= kAllocIncrement); 8985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) inuse_ = 0; 899926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) allocated_regions_ = HardenedSLL::null(); 9005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) free_area_ = NULL; 9015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) free_avail_ = 0; 902926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) free_list_.setValue(NULL); 903926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) entropy_ = entropy; 9045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 9055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 9065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) T* New() { 9075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Consult free list 9085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void* result; 909926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if (free_list_) { 910926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) result = free_list_.value(); 911926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) free_list_ = SLL_Next(free_list_, entropy_); 9125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } else { 9135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (free_avail_ < kAlignedSize) { 9145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Need more room 9155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) char* new_allocation = reinterpret_cast<char*>(MetaDataAlloc(kAllocIncrement)); 9165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (!new_allocation) 9175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CRASH(); 9185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 919926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL new_head = HardenedSLL::create(new_allocation); 920926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) SLL_SetNext(new_head, allocated_regions_, entropy_); 921926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) allocated_regions_ = new_head; 9225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) free_area_ = new_allocation + kAlignedSize; 9235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) free_avail_ = kAllocIncrement - kAlignedSize; 9245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 9255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) result = free_area_; 9265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) free_area_ += kAlignedSize; 9275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) free_avail_ -= kAlignedSize; 9285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 9295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) inuse_++; 9305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return reinterpret_cast<T*>(result); 9315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 9325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 9335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void Delete(T* p) { 934926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL new_head = HardenedSLL::create(p); 935926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) SLL_SetNext(new_head, free_list_, entropy_); 936926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) free_list_ = new_head; 9375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) inuse_--; 9385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 9395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 9405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int inuse() const { return inuse_; } 9415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 94253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)#if OS(DARWIN) 9435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) template <class Recorder> 9445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void recordAdministrativeRegions(Recorder& recorder, const RemoteMemoryReader& reader) 9455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) { 946926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) for (HardenedSLL adminAllocation = allocated_regions_; adminAllocation; adminAllocation.setValue(reader.nextEntryInHardenedLinkedList(reinterpret_cast<void**>(adminAllocation.value()), entropy_))) 947926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) recorder.recordRegion(reinterpret_cast<vm_address_t>(adminAllocation.value()), kAllocIncrement); 9485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 9495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 9505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)}; 9515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 9525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// ------------------------------------------------------------------------- 9535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Span - a contiguous run of pages 9545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// ------------------------------------------------------------------------- 9555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 9565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Type that can hold a page number 9575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)typedef uintptr_t PageID; 9585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 9595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Type that can hold the length of a run of pages 9605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)typedef uintptr_t Length; 9615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 9625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const Length kMaxValidPages = (~static_cast<Length>(0)) >> kPageShift; 9635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 9645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Convert byte size into pages. This won't overflow, but may return 9655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// an unreasonably large value if bytes is huge enough. 9665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static inline Length pages(size_t bytes) { 9675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return (bytes >> kPageShift) + 9685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ((bytes & (kPageSize - 1)) > 0 ? 1 : 0); 9695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 9705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 9715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Convert a user size into the number of bytes that will actually be 9725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// allocated 9735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static size_t AllocationSize(size_t bytes) { 9745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (bytes > kMaxSize) { 9755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Large object: we allocate an integral number of pages 9765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(bytes <= (kMaxValidPages << kPageShift)); 9775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return pages(bytes) << kPageShift; 9785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } else { 9795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Small object: find the size class to which it belongs 9805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return ByteSizeForClass(SizeClass(bytes)); 9815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 9825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 9835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 984926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)enum { 985926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) kSpanCookieBits = 10, 986926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) kSpanCookieMask = (1 << 10) - 1, 987926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) kSpanThisShift = 7 988926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)}; 989926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 990926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)static uint32_t spanValidationCookie; 991926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)static uint32_t spanInitializerCookie() 992926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles){ 993926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static uint32_t value = EntropySource<sizeof(uint32_t)>::value() & kSpanCookieMask; 994926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) spanValidationCookie = value; 995926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return value; 996926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)} 997926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 9985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Information kept for a span (a contiguous run of pages). 9995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)struct Span { 10005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) PageID start; // Starting page number 10015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Length length; // Number of pages in span 1002926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) Span* next(uintptr_t entropy) const { return XOR_MASK_PTR_WITH_KEY(m_next, this, entropy); } 1003926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) Span* remoteNext(const Span* remoteSpanPointer, uintptr_t entropy) const { return XOR_MASK_PTR_WITH_KEY(m_next, remoteSpanPointer, entropy); } 1004926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) Span* prev(uintptr_t entropy) const { return XOR_MASK_PTR_WITH_KEY(m_prev, this, entropy); } 1005926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) void setNext(Span* next, uintptr_t entropy) { m_next = XOR_MASK_PTR_WITH_KEY(next, this, entropy); } 1006926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) void setPrev(Span* prev, uintptr_t entropy) { m_prev = XOR_MASK_PTR_WITH_KEY(prev, this, entropy); } 1007926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1008926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)private: 1009926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) Span* m_next; // Used when in link list 1010926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) Span* m_prev; // Used when in link list 1011926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)public: 1012926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL objects; // Linked list of free objects 10135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) unsigned int free : 1; // Is the span free 10145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#ifndef NO_TCMALLOC_SAMPLES 10155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) unsigned int sample : 1; // Sampled object? 10165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 10175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) unsigned int sizeclass : 8; // Size-class for small objects (or 0) 10185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) unsigned int refcount : 11; // Number of non-free objects 10195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) bool decommitted : 1; 1020926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) void initCookie() 1021926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) { 1022926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) m_cookie = ((reinterpret_cast<uintptr_t>(this) >> kSpanThisShift) & kSpanCookieMask) ^ spanInitializerCookie(); 1023926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) } 1024926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) void clearCookie() { m_cookie = 0; } 1025926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) bool isValid() const 1026926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) { 1027926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return (((reinterpret_cast<uintptr_t>(this) >> kSpanThisShift) & kSpanCookieMask) ^ m_cookie) == spanValidationCookie; 1028926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) } 1029926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)private: 1030926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) uint32_t m_cookie : kSpanCookieBits; 10315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 10325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#undef SPAN_HISTORY 10335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#ifdef SPAN_HISTORY 10345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // For debugging, we can keep a log events per span 10355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int nexthistory; 10365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) char history[64]; 10375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int value[64]; 10385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 10395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)}; 10405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 10415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#define ASSERT_SPAN_COMMITTED(span) ASSERT(!span->decommitted) 10425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 10435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#ifdef SPAN_HISTORY 10445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void Event(Span* span, char op, int v = 0) { 10455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) span->history[span->nexthistory] = op; 10465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) span->value[span->nexthistory] = v; 10475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) span->nexthistory++; 10485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (span->nexthistory == sizeof(span->history)) span->nexthistory = 0; 10495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 10505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#else 10515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#define Event(s,o,v) ((void) 0) 10525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 10535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 10545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Allocator/deallocator for spans 10555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static PageHeapAllocator<Span> span_allocator; 10565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static Span* NewSpan(PageID p, Length len) { 10575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Span* result = span_allocator.New(); 10585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) memset(result, 0, sizeof(*result)); 10595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) result->start = p; 10605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) result->length = len; 1061926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) result->initCookie(); 10625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#ifdef SPAN_HISTORY 10635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) result->nexthistory = 0; 10645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 10655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return result; 10665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 10675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1068926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)static inline void DeleteSpan(Span* span) { 1069926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) RELEASE_ASSERT(span->isValid()); 1070926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#ifndef NDEBUG 1071926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // In debug mode, trash the contents of deleted Spans 1072926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) memset(span, 0x3f, sizeof(*span)); 1073926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#endif 1074926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) span->clearCookie(); 1075926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) span_allocator.Delete(span); 1076926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)} 1077926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1078926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)// ------------------------------------------------------------------------- 1079926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)// Doubly linked list of spans. 1080926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)// ------------------------------------------------------------------------- 1081926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1082926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)static inline void DLL_Init(Span* list, uintptr_t entropy) { 1083926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) list->setNext(list, entropy); 1084926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) list->setPrev(list, entropy); 1085926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)} 1086926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1087926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)static inline void DLL_Remove(Span* span, uintptr_t entropy) { 1088926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) span->prev(entropy)->setNext(span->next(entropy), entropy); 1089926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) span->next(entropy)->setPrev(span->prev(entropy), entropy); 1090926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) span->setPrev(NULL, entropy); 1091926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) span->setNext(NULL, entropy); 1092926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)} 1093926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1094926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)static ALWAYS_INLINE bool DLL_IsEmpty(const Span* list, uintptr_t entropy) { 1095926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return list->next(entropy) == list; 1096926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)} 1097926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1098926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)static int DLL_Length(const Span* list, uintptr_t entropy) { 1099926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) int result = 0; 1100926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) for (Span* s = list->next(entropy); s != list; s = s->next(entropy)) { 1101926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) result++; 1102926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) } 1103926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return result; 1104926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)} 1105926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1106926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#if 0 /* Not needed at the moment -- causes compiler warnings if not used */ 1107926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)static void DLL_Print(const char* label, const Span* list) { 1108926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) MESSAGE("%-10s %p:", label, list); 1109926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) for (const Span* s = list->next; s != list; s = s->next) { 1110926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) MESSAGE(" <%p,%u,%u>", s, s->start, s->length); 1111926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) } 1112926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) MESSAGE("\n"); 1113926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)} 1114926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#endif 1115926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1116926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)static inline void DLL_Prepend(Span* list, Span* span, uintptr_t entropy) { 1117926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) span->setNext(list->next(entropy), entropy); 1118926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) span->setPrev(list, entropy); 1119926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) list->next(entropy)->setPrev(span, entropy); 1120926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) list->setNext(span, entropy); 1121926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)} 1122926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1123926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)//------------------------------------------------------------------- 1124926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)// Data kept per size-class in central cache 1125926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)//------------------------------------------------------------------- 1126926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1127926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)class TCMalloc_Central_FreeList { 1128926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) public: 1129926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) void Init(size_t cl, uintptr_t entropy); 1130926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1131926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // These methods all do internal locking. 1132926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1133926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // Insert the specified range into the central freelist. N is the number of 1134926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // elements in the range. 1135926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) void InsertRange(HardenedSLL start, HardenedSLL end, int N); 1136926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1137926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // Returns the actual number of fetched elements into N. 1138926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) void RemoveRange(HardenedSLL* start, HardenedSLL* end, int *N); 1139926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1140926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // Returns the number of free objects in cache. 1141926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) size_t length() { 1142926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) SpinLockHolder h(&lock_); 1143926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return counter_; 1144926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) } 1145926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1146926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // Returns the number of free objects in the transfer cache. 1147926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) int tc_length() { 1148926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) SpinLockHolder h(&lock_); 1149926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return used_slots_ * num_objects_to_move[size_class_]; 1150926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) } 1151926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1152926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) template <class Finder, class Reader> 1153926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) void enumerateFreeObjects(Finder& finder, const Reader& reader, TCMalloc_Central_FreeList* remoteCentralFreeList) 1154926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) { 1155926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) { 1156926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static const ptrdiff_t emptyOffset = reinterpret_cast<const char*>(&empty_) - reinterpret_cast<const char*>(this); 1157926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) Span* remoteEmpty = reinterpret_cast<Span*>(reinterpret_cast<char*>(remoteCentralFreeList) + emptyOffset); 1158926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) Span* remoteSpan = nonempty_.remoteNext(remoteEmpty, entropy_); 1159926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) for (Span* span = reader(remoteEmpty); span && span != &empty_; remoteSpan = span->remoteNext(remoteSpan, entropy_), span = (remoteSpan ? reader(remoteSpan) : 0)) 1160926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ASSERT(!span->objects); 1161926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) } 1162926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1163926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ASSERT(!nonempty_.objects); 1164926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static const ptrdiff_t nonemptyOffset = reinterpret_cast<const char*>(&nonempty_) - reinterpret_cast<const char*>(this); 1165926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1166926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) Span* remoteNonempty = reinterpret_cast<Span*>(reinterpret_cast<char*>(remoteCentralFreeList) + nonemptyOffset); 1167926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) Span* remoteSpan = nonempty_.remoteNext(remoteNonempty, entropy_); 1168926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1169926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) for (Span* span = reader(remoteSpan); span && remoteSpan != remoteNonempty; remoteSpan = span->remoteNext(remoteSpan, entropy_), span = (remoteSpan ? reader(remoteSpan) : 0)) { 1170926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) for (HardenedSLL nextObject = span->objects; nextObject; nextObject.setValue(reader.nextEntryInHardenedLinkedList(reinterpret_cast<void**>(nextObject.value()), entropy_))) { 1171926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) finder.visit(nextObject.value()); 1172926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) } 1173926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) } 1174926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) } 1175926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1176926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) uintptr_t entropy() const { return entropy_; } 1177926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) private: 1178926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // REQUIRES: lock_ is held 1179926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // Remove object from cache and return. 1180926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // Return NULL if no free entries in cache. 1181926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL FetchFromSpans(); 1182926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1183926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // REQUIRES: lock_ is held 1184926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // Remove object from cache and return. Fetches 1185926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // from pageheap if cache is empty. Only returns 1186926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // NULL on allocation failure. 1187926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL FetchFromSpansSafe(); 1188926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1189926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // REQUIRES: lock_ is held 1190926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // Release a linked list of objects to spans. 1191926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // May temporarily release lock_. 1192926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) void ReleaseListToSpans(HardenedSLL start); 1193926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1194926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // REQUIRES: lock_ is held 1195926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // Release an object to spans. 1196926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // May temporarily release lock_. 1197926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ALWAYS_INLINE void ReleaseToSpans(HardenedSLL object); 1198926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1199926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // REQUIRES: lock_ is held 1200926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // Populate cache by fetching from the page heap. 1201926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // May temporarily release lock_. 1202926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ALWAYS_INLINE void Populate(); 1203926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1204926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // REQUIRES: lock is held. 1205926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // Tries to make room for a TCEntry. If the cache is full it will try to 1206926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // expand it at the cost of some other cache size. Return false if there is 1207926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // no space. 1208926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) bool MakeCacheSpace(); 1209926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1210926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // REQUIRES: lock_ for locked_size_class is held. 1211926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // Picks a "random" size class to steal TCEntry slot from. In reality it 1212926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // just iterates over the sizeclasses but does so without taking a lock. 1213926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // Returns true on success. 1214926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // May temporarily lock a "random" size class. 1215926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static ALWAYS_INLINE bool EvictRandomSizeClass(size_t locked_size_class, bool force); 1216926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1217926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // REQUIRES: lock_ is *not* held. 1218926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // Tries to shrink the Cache. If force is true it will relase objects to 1219926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // spans if it allows it to shrink the cache. Return false if it failed to 1220926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // shrink the cache. Decrements cache_size_ on succeess. 1221926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // May temporarily take lock_. If it takes lock_, the locked_size_class 1222926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // lock is released to the thread from holding two size class locks 1223926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // concurrently which could lead to a deadlock. 1224926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) bool ShrinkCache(int locked_size_class, bool force); 1225926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1226926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // This lock protects all the data members. cached_entries and cache_size_ 1227926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // may be looked at without holding the lock. 1228926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) SpinLock lock_; 1229926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1230926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // We keep linked lists of empty and non-empty spans. 1231926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) size_t size_class_; // My size class 1232926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) Span empty_; // Dummy header for list of empty spans 1233926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) Span nonempty_; // Dummy header for list of non-empty spans 1234926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) size_t counter_; // Number of free objects in cache entry 1235926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1236926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // Here we reserve space for TCEntry cache slots. Since one size class can 1237926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // end up getting all the TCEntries quota in the system we just preallocate 1238926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // sufficient number of entries here. 1239926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) TCEntry tc_slots_[kNumTransferEntries]; 1240926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1241926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // Number of currently used cached entries in tc_slots_. This variable is 1242926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // updated under a lock but can be read without one. 1243926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) int32_t used_slots_; 1244926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // The current number of slots for this size class. This is an 1245926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // adaptive value that is increased if there is lots of traffic 1246926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // on a given size class. 1247926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) int32_t cache_size_; 1248926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) uintptr_t entropy_; 1249926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)}; 1250926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1251926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#if COMPILER(CLANG) && defined(__has_warning) 1252926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#pragma clang diagnostic push 1253926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#if __has_warning("-Wunused-private-field") 1254926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#pragma clang diagnostic ignored "-Wunused-private-field" 1255926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#endif 1256926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#endif 1257926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1258926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)// Pad each CentralCache object to multiple of 64 bytes 1259926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)template <size_t SizeToPad> 1260926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)class TCMalloc_Central_FreeListPadded_Template : public TCMalloc_Central_FreeList { 1261926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)private: 1262926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) char pad[64 - SizeToPad]; 1263926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)}; 1264926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1265926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)// Zero-size specialization to avoid compiler error when TCMalloc_Central_FreeList happens 1266926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)// to be exactly 64 bytes. 1267926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)template <> class TCMalloc_Central_FreeListPadded_Template<0> : public TCMalloc_Central_FreeList { 1268926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)}; 1269926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1270926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)typedef TCMalloc_Central_FreeListPadded_Template<sizeof(TCMalloc_Central_FreeList) % 64> TCMalloc_Central_FreeListPadded; 1271926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1272926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#if COMPILER(CLANG) && defined(__has_warning) 1273926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#pragma clang diagnostic pop 1274926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#endif 1275926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1276926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#if OS(DARWIN) 1277926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)struct Span; 1278926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)class TCMalloc_PageHeap; 1279926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)class TCMalloc_ThreadCache; 1280926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)template <typename T> class PageHeapAllocator; 1281926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1282926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)class FastMallocZone { 1283926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)public: 1284926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static void init(); 1285926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1286926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static kern_return_t enumerate(task_t, void*, unsigned typeMmask, vm_address_t zoneAddress, memory_reader_t, vm_range_recorder_t); 1287926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static size_t goodSize(malloc_zone_t*, size_t size) { return size; } 1288926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static boolean_t check(malloc_zone_t*) { return true; } 1289926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static void print(malloc_zone_t*, boolean_t) { } 1290926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static void log(malloc_zone_t*, void*) { } 1291926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static void forceLock(malloc_zone_t*) { } 1292926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static void forceUnlock(malloc_zone_t*) { } 1293926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static void statistics(malloc_zone_t*, malloc_statistics_t* stats) { memset(stats, 0, sizeof(malloc_statistics_t)); } 1294926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1295926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)private: 1296926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) FastMallocZone(TCMalloc_PageHeap*, TCMalloc_ThreadCache**, TCMalloc_Central_FreeListPadded*, PageHeapAllocator<Span>*, PageHeapAllocator<TCMalloc_ThreadCache>*); 1297926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static size_t size(malloc_zone_t*, const void*); 1298926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static void* zoneMalloc(malloc_zone_t*, size_t); 1299926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static void* zoneCalloc(malloc_zone_t*, size_t numItems, size_t size); 1300926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static void zoneFree(malloc_zone_t*, void*); 1301926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static void* zoneRealloc(malloc_zone_t*, void*, size_t); 1302926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static void* zoneValloc(malloc_zone_t*, size_t) { LOG_ERROR("valloc is not supported"); return 0; } 1303926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static void zoneDestroy(malloc_zone_t*) { } 13045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1305926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) malloc_zone_t m_zone; 1306926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) TCMalloc_PageHeap* m_pageHeap; 1307926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) TCMalloc_ThreadCache** m_threadHeaps; 1308926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) TCMalloc_Central_FreeListPadded* m_centralCaches; 1309926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) PageHeapAllocator<Span>* m_spanAllocator; 1310926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) PageHeapAllocator<TCMalloc_ThreadCache>* m_pageHeapAllocator; 1311926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)}; 13125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1313926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#endif 13145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1315926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)// Even if we have support for thread-local storage in the compiler 1316926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)// and linker, the OS may not support it. We need to check that at 1317926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)// runtime. Right now, we have to keep a manual set of "bad" OSes. 1318926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#if defined(HAVE_TLS) 1319926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static bool kernel_supports_tls = false; // be conservative 1320926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static inline bool KernelSupportsTLS() { 1321926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return kernel_supports_tls; 13225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 1323926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# if !HAVE_DECL_UNAME // if too old for uname, probably too old for TLS 1324926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static void CheckIfKernelSupportsTLS() { 1325926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) kernel_supports_tls = false; 1326926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) } 1327926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# else 1328926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# include <sys/utsname.h> // DECL_UNAME checked for <sys/utsname.h> too 1329926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static void CheckIfKernelSupportsTLS() { 1330926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) struct utsname buf; 1331926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if (uname(&buf) != 0) { // should be impossible 1332926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) MESSAGE("uname failed assuming no TLS support (errno=%d)\n", errno); 1333926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) kernel_supports_tls = false; 1334926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) } else if (strcasecmp(buf.sysname, "linux") == 0) { 1335926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // The linux case: the first kernel to support TLS was 2.6.0 1336926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if (buf.release[0] < '2' && buf.release[1] == '.') // 0.x or 1.x 1337926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) kernel_supports_tls = false; 1338926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) else if (buf.release[0] == '2' && buf.release[1] == '.' && 1339926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) buf.release[2] >= '0' && buf.release[2] < '6' && 1340926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) buf.release[3] == '.') // 2.0 - 2.5 1341926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) kernel_supports_tls = false; 1342926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) else 1343926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) kernel_supports_tls = true; 1344926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) } else { // some other kernel, we'll be optimisitic 1345926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) kernel_supports_tls = true; 1346926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) } 1347926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // TODO(csilvers): VLOG(1) the tls status once we support RAW_VLOG 1348926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) } 1349926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# endif // HAVE_DECL_UNAME 1350926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#endif // HAVE_TLS 13515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1352926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)// __THROW is defined in glibc systems. It means, counter-intuitively, 1353926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)// "This function will never throw an exception." It's an optional 1354926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)// optimization tool, but we may need to use it to match glibc prototypes. 1355926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#ifndef __THROW // I guess we're not on a glibc system 1356926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# define __THROW // __THROW is just an optimization, so ok to make it "" 13575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 13585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 13595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// ------------------------------------------------------------------------- 13605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Stack traces kept for sampled allocations 13615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// The following state is protected by pageheap_lock_. 13625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// ------------------------------------------------------------------------- 13635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 13645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// size/depth are made the same size as a pointer so that some generic 13655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// code below can conveniently cast them back and forth to void*. 13665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const int kMaxStackDepth = 31; 13675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)struct StackTrace { 13685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) uintptr_t size; // Size of object 13695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) uintptr_t depth; // Number of PC values stored in array below 13705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void* stack[kMaxStackDepth]; 13715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)}; 13725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static PageHeapAllocator<StackTrace> stacktrace_allocator; 13735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static Span sampled_objects; 13745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 13755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// ------------------------------------------------------------------------- 13765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Map from page-id to per-page data 13775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// ------------------------------------------------------------------------- 13785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 13795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// We use PageMap2<> for 32-bit and PageMap3<> for 64-bit machines. 13805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// We also use a simple one-level cache for hot PageID-to-sizeclass mappings, 13815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// because sometimes the sizeclass is all the information we need. 13825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 13835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Selector class -- general selector uses 3-level map 13845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)template <int BITS> class MapSelector { 13855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) public: 13865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) typedef TCMalloc_PageMap3<BITS-kPageShift> Type; 13875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) typedef PackedCache<BITS, uint64_t> CacheType; 13885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)}; 13895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 13905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if CPU(X86_64) 139102772c6a72f1ee0b226341a4f4439970c29fc861Ben Murdoch// On all known X86-64 platforms, the upper 16 bits are always unused and therefore 13925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// can be excluded from the PageMap key. 13935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// See http://en.wikipedia.org/wiki/X86-64#Virtual_address_space_details 13945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 13955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const size_t kBitsUnusedOn64Bit = 16; 13965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#else 13975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const size_t kBitsUnusedOn64Bit = 0; 13985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 13995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 14005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// A three-level map for 64-bit machines 14015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)template <> class MapSelector<64> { 14025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) public: 14035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) typedef TCMalloc_PageMap3<64 - kPageShift - kBitsUnusedOn64Bit> Type; 14045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) typedef PackedCache<64, uint64_t> CacheType; 14055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)}; 14065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 14075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// A two-level map for 32-bit machines 14085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)template <> class MapSelector<32> { 14095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) public: 14105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) typedef TCMalloc_PageMap2<32 - kPageShift> Type; 14115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) typedef PackedCache<32 - kPageShift, uint16_t> CacheType; 14125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)}; 14135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 14145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// ------------------------------------------------------------------------- 14155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Page-level allocator 14165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// * Eager coalescing 14175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 14185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Heap for page-level allocation. We allow allocating and freeing a 14195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// contiguous runs of pages (called a "span"). 14205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// ------------------------------------------------------------------------- 14215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 14225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 14235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// The page heap maintains a free list for spans that are no longer in use by 14245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// the central cache or any thread caches. We use a background thread to 14255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// periodically scan the free list and release a percentage of it back to the OS. 14265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 14275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// If free_committed_pages_ exceeds kMinimumFreeCommittedPageCount, the 14285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// background thread: 14295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// - wakes up 14305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// - pauses for kScavengeDelayInSeconds 14315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// - returns to the OS a percentage of the memory that remained unused during 14325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// that pause (kScavengePercentage * min_free_committed_pages_since_last_scavenge_) 14335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// The goal of this strategy is to reduce memory pressure in a timely fashion 14345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// while avoiding thrashing the OS allocator. 14355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 14365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Time delay before the page heap scavenger will consider returning pages to 14375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// the OS. 14385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const int kScavengeDelayInSeconds = 2; 14395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 14405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Approximate percentage of free committed pages to return to the OS in one 14415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// scavenge. 14425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const float kScavengePercentage = .5f; 14435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 14445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// number of span lists to keep spans in when memory is returned. 14455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const int kMinSpanListsWithSpans = 32; 14465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 14475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Number of free committed pages that we want to keep around. The minimum number of pages used when there 14485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// is 1 span in each of the first kMinSpanListsWithSpans spanlists. Currently 528 pages. 14495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static const size_t kMinimumFreeCommittedPageCount = kMinSpanListsWithSpans * ((1.0f+kMinSpanListsWithSpans) / 2.0f); 14505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 14515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 14525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 14535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static SpinLock pageheap_lock = SPINLOCK_INITIALIZER; 14545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 14555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class TCMalloc_PageHeap { 14565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) public: 14575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void init(); 14585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 14595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Allocate a run of "n" pages. Returns zero if out of memory. 14605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Span* New(Length n); 14615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 14625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Delete the span "[p, p+n-1]". 14635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // REQUIRES: span was returned by earlier call to New() and 14645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // has not yet been deleted. 14655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void Delete(Span* span); 14665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 14675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Mark an allocated span as being used for small objects of the 14685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // specified size-class. 14695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // REQUIRES: span was returned by an earlier call to New() 14705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // and has not yet been deleted. 14715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void RegisterSizeClass(Span* span, size_t sc); 14725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 14735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Split an allocated span into two spans: one of length "n" pages 14745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // followed by another span of length "span->length - n" pages. 14755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Modifies "*span" to point to the first span of length "n" pages. 14765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Returns a pointer to the second span. 14775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // 14785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // REQUIRES: "0 < n < span->length" 14795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // REQUIRES: !span->free 14805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // REQUIRES: span->sizeclass == 0 14815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Span* Split(Span* span, Length n); 14825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 14835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Return the descriptor for the specified page. 14845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) inline Span* GetDescriptor(PageID p) const { 14855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return reinterpret_cast<Span*>(pagemap_.get(p)); 14865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 14875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 14885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) inline Span* GetDescriptorEnsureSafe(PageID p) 14895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) { 14905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pagemap_.Ensure(p, 1); 14915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return GetDescriptor(p); 14925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 149302772c6a72f1ee0b226341a4f4439970c29fc861Ben Murdoch 14945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t ReturnedBytes() const; 14955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 14965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Return number of bytes allocated from system 14975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) inline uint64_t SystemBytes() const { return system_bytes_; } 14985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 14995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Return number of free bytes in heap 15005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) uint64_t FreeBytes() const { 15015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return (static_cast<uint64_t>(free_pages_) << kPageShift); 15025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 15035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 15045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) bool Check(); 15055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t CheckList(Span* list, Length min_pages, Length max_pages, bool decommitted); 15065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 15075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Release all pages on the free list for reuse by the OS: 15085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void ReleaseFreePages(); 15095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void ReleaseFreeList(Span*, Span*); 15105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 15115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Return 0 if we have no information, or else the correct sizeclass for p. 15125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Reads and writes to pagemap_cache_ do not require locking. 15135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // The entries are 64 bits on 64-bit hardware and 16 bits on 15145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // 32-bit hardware, and we don't mind raciness as long as each read of 15155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // an entry yields a valid entry, not a partially updated entry. 15165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t GetSizeClassIfCached(PageID p) const { 15175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return pagemap_cache_.GetOrDefault(p, 0); 15185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 15195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void CacheSizeClass(PageID p, size_t cl) const { pagemap_cache_.Put(p, cl); } 15205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 15215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) private: 15225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Pick the appropriate map and cache types based on pointer size 15235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) typedef MapSelector<8*sizeof(uintptr_t)>::Type PageMap; 15245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) typedef MapSelector<8*sizeof(uintptr_t)>::CacheType PageMapCache; 15255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) PageMap pagemap_; 15265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) mutable PageMapCache pagemap_cache_; 15275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 15285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // We segregate spans of a given size into two circular linked 15295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // lists: one for normal spans, and one for spans whose memory 15305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // has been returned to the system. 15315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) struct SpanList { 15325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Span normal; 15335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Span returned; 15345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) }; 15355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 15365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // List of free spans of length >= kMaxPages 15375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) SpanList large_; 15385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 15395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Array mapping from span length to a doubly linked list of free spans 15405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) SpanList free_[kMaxPages]; 15415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 15425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Number of pages kept in free lists 15435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) uintptr_t free_pages_; 15445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1545926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // Used for hardening 1546926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) uintptr_t entropy_; 1547926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 15485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Bytes allocated from system 15495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) uint64_t system_bytes_; 15505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 15515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 15525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Number of pages kept in free lists that are still committed. 15535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Length free_committed_pages_; 15545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 15555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Minimum number of free committed pages since last scavenge. (Can be 0 if 15565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // we've committed new pages since the last scavenge.) 15575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Length min_free_committed_pages_since_last_scavenge_; 15585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 15595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 15605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) bool GrowHeap(Length n); 15615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 15625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // REQUIRES span->length >= n 15635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Remove span from its free list, and move any leftover part of 15645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // span into appropriate free lists. Also update "span" to have 15655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // length exactly "n" and mark it as non-free so it can be returned 15665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // to the client. 15675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // 15685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // "released" is true iff "span" was found on a "returned" list. 15695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void Carve(Span* span, Length n, bool released); 15705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 15715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void RecordSpan(Span* span) { 15725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pagemap_.set(span->start, span); 15735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (span->length > 1) { 15745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pagemap_.set(span->start + span->length - 1, span); 15755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 15765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 157702772c6a72f1ee0b226341a4f4439970c29fc861Ben Murdoch 15785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Allocate a large span of length == n. If successful, returns a 15795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // span of exactly the specified length. Else, returns NULL. 15805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Span* AllocLarge(Length n); 15815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 15825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if !USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 15835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Incrementally release some memory to the system. 15845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // IncrementalScavenge(n) is called whenever n pages are freed. 15855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void IncrementalScavenge(Length n); 15865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 15875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 15885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Number of pages to deallocate before doing more scavenging 15895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int64_t scavenge_counter_; 15905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 15915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Index of last free list we scavenged 15925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t scavenge_index_; 159302772c6a72f1ee0b226341a4f4439970c29fc861Ben Murdoch 159453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)#if OS(DARWIN) 15955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) friend class FastMallocZone; 15965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 15975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 15985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 15995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void initializeScavenger(); 16005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ALWAYS_INLINE void signalScavenger(); 16015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void scavenge(); 16025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ALWAYS_INLINE bool shouldScavenge() const; 16035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 16045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if HAVE(DISPATCH_H) || OS(WINDOWS) 16055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void periodicScavenge(); 16065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ALWAYS_INLINE bool isScavengerSuspended(); 16075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ALWAYS_INLINE void scheduleScavenger(); 16085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ALWAYS_INLINE void rescheduleScavenger(); 16095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ALWAYS_INLINE void suspendScavenger(); 16105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 16115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 16125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if HAVE(DISPATCH_H) 16135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) dispatch_queue_t m_scavengeQueue; 16145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) dispatch_source_t m_scavengeTimer; 16155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) bool m_scavengingSuspended; 16165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#elif OS(WINDOWS) 16175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) static void CALLBACK scavengerTimerFired(void*, BOOLEAN); 16185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) HANDLE m_scavengeQueueTimer; 161902772c6a72f1ee0b226341a4f4439970c29fc861Ben Murdoch#else 16205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) static NO_RETURN_WITH_VALUE void* runScavengerThread(void*); 16215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) NO_RETURN void scavengerThread(); 16225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 16235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Keeps track of whether the background thread is actively scavenging memory every kScavengeDelayInSeconds, or 16245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // it's blocked waiting for more pages to be deleted. 16255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) bool m_scavengeThreadActive; 16265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 16275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_mutex_t m_scavengeMutex; 16285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_cond_t m_scavengeCondition; 16295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 16305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 16315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 16325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)}; 16335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 16345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void TCMalloc_PageHeap::init() 16355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 16365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pagemap_.init(MetaDataAlloc); 16375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pagemap_cache_ = PageMapCache(0); 16385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) free_pages_ = 0; 16395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) system_bytes_ = 0; 1640926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) entropy_ = HARDENING_ENTROPY; 16415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 16425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 16435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) free_committed_pages_ = 0; 16445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) min_free_committed_pages_since_last_scavenge_ = 0; 16455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 16465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 16475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) scavenge_counter_ = 0; 16485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Start scavenging at kMaxPages list 16495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) scavenge_index_ = kMaxPages-1; 16505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) COMPILE_ASSERT(kNumClasses <= (1 << PageMapCache::kValuebits), valuebits); 1651926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Init(&large_.normal, entropy_); 1652926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Init(&large_.returned, entropy_); 16535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (size_t i = 0; i < kMaxPages; i++) { 1654926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Init(&free_[i].normal, entropy_); 1655926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Init(&free_[i].returned, entropy_); 16565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 16575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 16585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 16595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) initializeScavenger(); 16605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 16615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 16625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 16635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 16645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 16655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if HAVE(DISPATCH_H) 16665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 16675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void TCMalloc_PageHeap::initializeScavenger() 16685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 16695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_scavengeQueue = dispatch_queue_create("com.apple.JavaScriptCore.FastMallocSavenger", NULL); 16705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_scavengeTimer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, m_scavengeQueue); 1671926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) uint64_t scavengeDelayInNanoseconds = kScavengeDelayInSeconds * NSEC_PER_SEC; 1672926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) dispatch_time_t startTime = dispatch_time(DISPATCH_TIME_NOW, scavengeDelayInNanoseconds); 1673926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) dispatch_source_set_timer(m_scavengeTimer, startTime, scavengeDelayInNanoseconds, scavengeDelayInNanoseconds / 10); 16745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) dispatch_source_set_event_handler(m_scavengeTimer, ^{ periodicScavenge(); }); 16755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_scavengingSuspended = true; 16765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 16775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 16785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)ALWAYS_INLINE bool TCMalloc_PageHeap::isScavengerSuspended() 16795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 16805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(pageheap_lock.IsHeld()); 16815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return m_scavengingSuspended; 16825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 16835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 16845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)ALWAYS_INLINE void TCMalloc_PageHeap::scheduleScavenger() 16855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 16865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(pageheap_lock.IsHeld()); 16875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_scavengingSuspended = false; 16885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) dispatch_resume(m_scavengeTimer); 16895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 16905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 16915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)ALWAYS_INLINE void TCMalloc_PageHeap::rescheduleScavenger() 16925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 16935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Nothing to do here for libdispatch. 16945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 16955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 16965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)ALWAYS_INLINE void TCMalloc_PageHeap::suspendScavenger() 16975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 16985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(pageheap_lock.IsHeld()); 16995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_scavengingSuspended = true; 17005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) dispatch_suspend(m_scavengeTimer); 17015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 17025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 17035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#elif OS(WINDOWS) 17045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 17055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void TCMalloc_PageHeap::scavengerTimerFired(void* context, BOOLEAN) 17065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 17075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) static_cast<TCMalloc_PageHeap*>(context)->periodicScavenge(); 17085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 17095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 17105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void TCMalloc_PageHeap::initializeScavenger() 17115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 17125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_scavengeQueueTimer = 0; 17135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 17145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 17155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)ALWAYS_INLINE bool TCMalloc_PageHeap::isScavengerSuspended() 17165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 17175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(pageheap_lock.IsHeld()); 17185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return !m_scavengeQueueTimer; 17195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 17205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 17215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)ALWAYS_INLINE void TCMalloc_PageHeap::scheduleScavenger() 17225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 17235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // We need to use WT_EXECUTEONLYONCE here and reschedule the timer, because 17245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Windows will fire the timer event even when the function is already running. 17255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(pageheap_lock.IsHeld()); 17265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CreateTimerQueueTimer(&m_scavengeQueueTimer, 0, scavengerTimerFired, this, kScavengeDelayInSeconds * 1000, 0, WT_EXECUTEONLYONCE); 17275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 17285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 17295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)ALWAYS_INLINE void TCMalloc_PageHeap::rescheduleScavenger() 17305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 17315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // We must delete the timer and create it again, because it is not possible to retrigger a timer on Windows. 17325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) suspendScavenger(); 17335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) scheduleScavenger(); 17345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 17355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 17365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)ALWAYS_INLINE void TCMalloc_PageHeap::suspendScavenger() 17375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 17385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(pageheap_lock.IsHeld()); 17395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) HANDLE scavengeQueueTimer = m_scavengeQueueTimer; 17405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_scavengeQueueTimer = 0; 17415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) DeleteTimerQueueTimer(0, scavengeQueueTimer, 0); 17425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 17435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 17445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#else 17455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 17465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void TCMalloc_PageHeap::initializeScavenger() 17475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 17485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Create a non-recursive mutex. 17495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if !defined(PTHREAD_MUTEX_NORMAL) || PTHREAD_MUTEX_NORMAL == PTHREAD_MUTEX_DEFAULT 17505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_mutex_init(&m_scavengeMutex, 0); 17515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#else 17525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_mutexattr_t attr; 17535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_mutexattr_init(&attr); 17545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL); 17555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 17565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_mutex_init(&m_scavengeMutex, &attr); 17575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 17585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_mutexattr_destroy(&attr); 17595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 17605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 17615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_cond_init(&m_scavengeCondition, 0); 17625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_scavengeThreadActive = true; 17635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_t thread; 17645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_create(&thread, 0, runScavengerThread, this); 17655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 17665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 17675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void* TCMalloc_PageHeap::runScavengerThread(void* context) 17685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 17695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) static_cast<TCMalloc_PageHeap*>(context)->scavengerThread(); 177093ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles)#if COMPILER(MSVC) 177193ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) // Without this, Visual Studio will complain that this method does not return a value. 17725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 0; 17735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 17745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 17755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 17765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)ALWAYS_INLINE void TCMalloc_PageHeap::signalScavenger() 17775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 177802772c6a72f1ee0b226341a4f4439970c29fc861Ben Murdoch // shouldScavenge() should be called only when the pageheap_lock spinlock is held, additionally, 17795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // m_scavengeThreadActive is only set to false whilst pageheap_lock is held. The caller must ensure this is 17805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // taken prior to calling this method. If the scavenger thread is sleeping and shouldScavenge() indicates there 17815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // is memory to free the scavenger thread is signalled to start. 17825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(pageheap_lock.IsHeld()); 17835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (!m_scavengeThreadActive && shouldScavenge()) 17845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_cond_signal(&m_scavengeCondition); 17855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 17865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 17875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 17885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 17895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void TCMalloc_PageHeap::scavenge() 17905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 17915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t pagesToRelease = min_free_committed_pages_since_last_scavenge_ * kScavengePercentage; 17925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t targetPageCount = std::max<size_t>(kMinimumFreeCommittedPageCount, free_committed_pages_ - pagesToRelease); 17935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 17945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Length lastFreeCommittedPages = free_committed_pages_; 17955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) while (free_committed_pages_ > targetPageCount) { 17965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(Check()); 17975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (int i = kMaxPages; i > 0 && free_committed_pages_ >= targetPageCount; i--) { 17985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) SpanList* slist = (static_cast<size_t>(i) == kMaxPages) ? &large_ : &free_[i]; 179902772c6a72f1ee0b226341a4f4439970c29fc861Ben Murdoch // If the span size is bigger than kMinSpanListsWithSpans pages return all the spans in the list, else return all but 1 span. 18005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Return only 50% of a spanlist at a time so spans of size 1 are not the only ones left. 1801926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) size_t length = DLL_Length(&slist->normal, entropy_); 18025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t numSpansToReturn = (i > kMinSpanListsWithSpans) ? length : length / 2; 1803926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) for (int j = 0; static_cast<size_t>(j) < numSpansToReturn && !DLL_IsEmpty(&slist->normal, entropy_) && free_committed_pages_ > targetPageCount; j++) { 1804926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) Span* s = slist->normal.prev(entropy_); 1805926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Remove(s, entropy_); 18065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(!s->decommitted); 18075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (!s->decommitted) { 18085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TCMalloc_SystemRelease(reinterpret_cast<void*>(s->start << kPageShift), 18095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) static_cast<size_t>(s->length << kPageShift)); 18105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(free_committed_pages_ >= s->length); 18115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) free_committed_pages_ -= s->length; 18125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) s->decommitted = true; 18135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 1814926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Prepend(&slist->returned, s, entropy_); 18155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 18165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 18175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 18185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (lastFreeCommittedPages == free_committed_pages_) 18195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) break; 18205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) lastFreeCommittedPages = free_committed_pages_; 18215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 18225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 18235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) min_free_committed_pages_since_last_scavenge_ = free_committed_pages_; 18245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 18255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 182602772c6a72f1ee0b226341a4f4439970c29fc861Ben MurdochALWAYS_INLINE bool TCMalloc_PageHeap::shouldScavenge() const 18275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 182802772c6a72f1ee0b226341a4f4439970c29fc861Ben Murdoch return free_committed_pages_ > kMinimumFreeCommittedPageCount; 18295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 18305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 18315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 18325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 18335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)inline Span* TCMalloc_PageHeap::New(Length n) { 18345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(Check()); 18355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(n > 0); 18365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 18375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Find first size >= n that has a non-empty list 18385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (Length s = n; s < kMaxPages; s++) { 18395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Span* ll = NULL; 18405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) bool released = false; 1841926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if (!DLL_IsEmpty(&free_[s].normal, entropy_)) { 18425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Found normal span 18435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ll = &free_[s].normal; 1844926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) } else if (!DLL_IsEmpty(&free_[s].returned, entropy_)) { 18455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Found returned span; reallocate it 18465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ll = &free_[s].returned; 18475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) released = true; 18485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } else { 18495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Keep looking in larger classes 18505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) continue; 18515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 18525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1853926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) Span* result = ll->next(entropy_); 18545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Carve(result, n, released); 18555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 18565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // The newly allocated memory is from a span that's in the normal span list (already committed). Update the 18575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // free committed pages count. 18585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(free_committed_pages_ >= n); 18595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) free_committed_pages_ -= n; 186002772c6a72f1ee0b226341a4f4439970c29fc861Ben Murdoch if (free_committed_pages_ < min_free_committed_pages_since_last_scavenge_) 18615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) min_free_committed_pages_since_last_scavenge_ = free_committed_pages_; 18625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 18635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(Check()); 18645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) free_pages_ -= n; 18655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return result; 18665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 18675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 18685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Span* result = AllocLarge(n); 18695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (result != NULL) { 18705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT_SPAN_COMMITTED(result); 18715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return result; 18725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 18735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 18745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Grow the heap and try again 18755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (!GrowHeap(n)) { 18765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(Check()); 18775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return NULL; 18785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 18795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 18805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return New(n); 18815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 18825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 18835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)Span* TCMalloc_PageHeap::AllocLarge(Length n) { 18845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // find the best span (closest to n in size). 18855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // The following loops implements address-ordered best-fit. 18865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) bool from_released = false; 18875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Span *best = NULL; 18885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 18895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Search through normal list 1890926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) for (Span* span = large_.normal.next(entropy_); 18915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) span != &large_.normal; 1892926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) span = span->next(entropy_)) { 18935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (span->length >= n) { 18945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if ((best == NULL) 18955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) || (span->length < best->length) 18965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) || ((span->length == best->length) && (span->start < best->start))) { 18975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) best = span; 18985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) from_released = false; 18995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 19005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 19015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 19025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 19035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Search through released list in case it has a better fit 1904926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) for (Span* span = large_.returned.next(entropy_); 19055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) span != &large_.returned; 1906926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) span = span->next(entropy_)) { 19075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (span->length >= n) { 19085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if ((best == NULL) 19095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) || (span->length < best->length) 19105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) || ((span->length == best->length) && (span->start < best->start))) { 19115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) best = span; 19125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) from_released = true; 19135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 19145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 19155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 19165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 19175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (best != NULL) { 19185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Carve(best, n, from_released); 19195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 19205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // The newly allocated memory is from a span that's in the normal span list (already committed). Update the 19215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // free committed pages count. 19225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(free_committed_pages_ >= n); 19235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) free_committed_pages_ -= n; 19245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (free_committed_pages_ < min_free_committed_pages_since_last_scavenge_) 19255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) min_free_committed_pages_since_last_scavenge_ = free_committed_pages_; 19265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 19275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(Check()); 19285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) free_pages_ -= n; 19295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return best; 19305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 19315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return NULL; 19325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 19335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 19345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)Span* TCMalloc_PageHeap::Split(Span* span, Length n) { 19355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(0 < n); 19365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(n < span->length); 19375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(!span->free); 19385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(span->sizeclass == 0); 19395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Event(span, 'T', n); 19405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 19415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const Length extra = span->length - n; 19425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Span* leftover = NewSpan(span->start + n, extra); 19435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Event(leftover, 'U', extra); 19445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) RecordSpan(leftover); 19455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pagemap_.set(span->start + n - 1, span); // Update map from pageid to span 19465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) span->length = n; 19475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 19485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return leftover; 19495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 19505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 19515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)inline void TCMalloc_PageHeap::Carve(Span* span, Length n, bool released) { 19525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(n > 0); 1953926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Remove(span, entropy_); 19545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) span->free = 0; 19555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Event(span, 'A', n); 19565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 19575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (released) { 19585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // If the span chosen to carve from is decommited, commit the entire span at once to avoid committing spans 1 page at a time. 19595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(span->decommitted); 19605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TCMalloc_SystemCommit(reinterpret_cast<void*>(span->start << kPageShift), static_cast<size_t>(span->length << kPageShift)); 19615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) span->decommitted = false; 19625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 19635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) free_committed_pages_ += span->length; 19645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 19655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 196602772c6a72f1ee0b226341a4f4439970c29fc861Ben Murdoch 19675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const int extra = static_cast<int>(span->length - n); 19685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(extra >= 0); 19695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (extra > 0) { 19705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Span* leftover = NewSpan(span->start + n, extra); 19715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) leftover->free = 1; 19725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) leftover->decommitted = false; 19735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Event(leftover, 'S', extra); 19745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) RecordSpan(leftover); 19755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 19765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Place leftover span on appropriate free list 19775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) SpanList* listpair = (static_cast<size_t>(extra) < kMaxPages) ? &free_[extra] : &large_; 19785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Span* dst = &listpair->normal; 1979926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Prepend(dst, leftover, entropy_); 19805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 19815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) span->length = n; 19825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pagemap_.set(span->start + n - 1, span); 19835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 19845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 19855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 19865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static ALWAYS_INLINE void mergeDecommittedStates(Span* destination, Span* other) 19875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 19885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (destination->decommitted && !other->decommitted) { 19895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TCMalloc_SystemRelease(reinterpret_cast<void*>(other->start << kPageShift), 19905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) static_cast<size_t>(other->length << kPageShift)); 19915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } else if (other->decommitted && !destination->decommitted) { 19925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TCMalloc_SystemRelease(reinterpret_cast<void*>(destination->start << kPageShift), 19935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) static_cast<size_t>(destination->length << kPageShift)); 19945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) destination->decommitted = true; 19955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 19965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 19975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 19985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)inline void TCMalloc_PageHeap::Delete(Span* span) { 19995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(Check()); 20005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(!span->free); 20015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(span->length > 0); 20025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(GetDescriptor(span->start) == span); 20035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(GetDescriptor(span->start + span->length - 1) == span); 20045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) span->sizeclass = 0; 20055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#ifndef NO_TCMALLOC_SAMPLES 20065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) span->sample = 0; 20075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 20085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 20095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Coalesce -- we guarantee that "p" != 0, so no bounds checking 20105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // necessary. We do not bother resetting the stale pagemap 20115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // entries for the pieces we are merging together because we only 20125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // care about the pagemap entries for the boundaries. 20135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 20145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Track the total size of the neighboring free spans that are committed. 20155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Length neighboringCommittedSpansLength = 0; 20165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 20175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const PageID p = span->start; 20185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const Length n = span->length; 20195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Span* prev = GetDescriptor(p-1); 20205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (prev != NULL && prev->free) { 20215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Merge preceding span into this span 20225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(prev->start + prev->length == p); 20235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const Length len = prev->length; 20245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 20255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (!prev->decommitted) 20265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) neighboringCommittedSpansLength += len; 20275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 20285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) mergeDecommittedStates(span, prev); 2029926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Remove(prev, entropy_); 20305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) DeleteSpan(prev); 20315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) span->start -= len; 20325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) span->length += len; 20335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pagemap_.set(span->start, span); 20345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Event(span, 'L', len); 20355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 20365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Span* next = GetDescriptor(p+n); 20375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (next != NULL && next->free) { 20385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Merge next span into this span 20395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(next->start == p+n); 20405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const Length len = next->length; 20415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 20425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (!next->decommitted) 20435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) neighboringCommittedSpansLength += len; 20445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 20455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) mergeDecommittedStates(span, next); 2046926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Remove(next, entropy_); 20475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) DeleteSpan(next); 20485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) span->length += len; 20495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pagemap_.set(span->start + span->length - 1, span); 20505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Event(span, 'R', len); 20515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 20525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 20535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Event(span, 'D', span->length); 20545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) span->free = 1; 20555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (span->decommitted) { 20565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (span->length < kMaxPages) 2057926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Prepend(&free_[span->length].returned, span, entropy_); 20585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) else 2059926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Prepend(&large_.returned, span, entropy_); 20605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } else { 20615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (span->length < kMaxPages) 2062926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Prepend(&free_[span->length].normal, span, entropy_); 20635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) else 2064926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Prepend(&large_.normal, span, entropy_); 20655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 20665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) free_pages_ += n; 20675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 20685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 20695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (span->decommitted) { 20705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // If the merged span is decommitted, that means we decommitted any neighboring spans that were 20715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // committed. Update the free committed pages count. 20725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) free_committed_pages_ -= neighboringCommittedSpansLength; 20735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (free_committed_pages_ < min_free_committed_pages_since_last_scavenge_) 20745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) min_free_committed_pages_since_last_scavenge_ = free_committed_pages_; 20755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } else { 20765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // If the merged span remains committed, add the deleted span's size to the free committed pages count. 20775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) free_committed_pages_ += n; 20785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 20795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 20805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Make sure the scavenge thread becomes active if we have enough freed pages to release some back to the system. 20815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) signalScavenger(); 20825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#else 20835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) IncrementalScavenge(n); 20845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 20855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 20865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(Check()); 20875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 20885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 20895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if !USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 20905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void TCMalloc_PageHeap::IncrementalScavenge(Length n) { 20915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Fast path; not yet time to release memory 20925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) scavenge_counter_ -= n; 20935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (scavenge_counter_ >= 0) return; // Not yet time to scavenge 20945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 20955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // If there is nothing to release, wait for so many pages before 20965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // scavenging again. With 4K pages, this comes to 16MB of memory. 20975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) static const size_t kDefaultReleaseDelay = 1 << 8; 20985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 20995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Find index of free list to scavenge 21005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t index = scavenge_index_ + 1; 2101926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) uintptr_t entropy = entropy_; 21025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (size_t i = 0; i < kMaxPages+1; i++) { 21035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (index > kMaxPages) index = 0; 21045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) SpanList* slist = (index == kMaxPages) ? &large_ : &free_[index]; 2105926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if (!DLL_IsEmpty(&slist->normal, entropy)) { 21065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Release the last span on the normal portion of this list 2107926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) Span* s = slist->normal.prev(entropy); 2108926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Remove(s, entropy_); 21095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TCMalloc_SystemRelease(reinterpret_cast<void*>(s->start << kPageShift), 21105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) static_cast<size_t>(s->length << kPageShift)); 21115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) s->decommitted = true; 2112926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Prepend(&slist->returned, s, entropy); 21135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 21145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) scavenge_counter_ = std::max<size_t>(64UL, std::min<size_t>(kDefaultReleaseDelay, kDefaultReleaseDelay - (free_pages_ / kDefaultReleaseDelay))); 21155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2116926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if (index == kMaxPages && !DLL_IsEmpty(&slist->normal, entropy)) 21175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) scavenge_index_ = index - 1; 21185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) else 21195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) scavenge_index_ = index; 21205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return; 21215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 21225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) index++; 21235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 21245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 21255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Nothing to scavenge, delay for a while 21265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) scavenge_counter_ = kDefaultReleaseDelay; 21275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 21285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 21295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 21305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void TCMalloc_PageHeap::RegisterSizeClass(Span* span, size_t sc) { 21315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Associate span object with all interior pages as well 21325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(!span->free); 21335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(GetDescriptor(span->start) == span); 21345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(GetDescriptor(span->start+span->length-1) == span); 21355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Event(span, 'C', sc); 21365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) span->sizeclass = static_cast<unsigned int>(sc); 21375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (Length i = 1; i < span->length-1; i++) { 21385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pagemap_.set(span->start+i, span); 21395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 21405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 214102772c6a72f1ee0b226341a4f4439970c29fc861Ben Murdoch 21425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)size_t TCMalloc_PageHeap::ReturnedBytes() const { 21435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t result = 0; 21445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (unsigned s = 0; s < kMaxPages; s++) { 2145926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) const int r_length = DLL_Length(&free_[s].returned, entropy_); 21465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) unsigned r_pages = s * r_length; 21475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) result += r_pages << kPageShift; 21485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 214902772c6a72f1ee0b226341a4f4439970c29fc861Ben Murdoch 2150926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) for (Span* s = large_.returned.next(entropy_); s != &large_.returned; s = s->next(entropy_)) 21515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) result += s->length << kPageShift; 21525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return result; 21535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 21545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 21555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)bool TCMalloc_PageHeap::GrowHeap(Length n) { 21565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(kMaxPages >= kMinSystemAlloc); 21575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (n > kMaxValidPages) return false; 21585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Length ask = (n>kMinSystemAlloc) ? n : static_cast<Length>(kMinSystemAlloc); 21595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t actual_size; 21605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void* ptr = TCMalloc_SystemAlloc(ask << kPageShift, &actual_size, kPageSize); 21615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (ptr == NULL) { 21625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (n < ask) { 21635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Try growing just "n" pages 21645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ask = n; 21655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ptr = TCMalloc_SystemAlloc(ask << kPageShift, &actual_size, kPageSize); 21665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 21675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (ptr == NULL) return false; 21685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 21695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ask = actual_size >> kPageShift; 21705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 21715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) uint64_t old_system_bytes = system_bytes_; 21725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) system_bytes_ += (ask << kPageShift); 21735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; 21745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(p > 0); 21755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 21765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // If we have already a lot of pages allocated, just pre allocate a bunch of 21775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // memory for the page map. This prevents fragmentation by pagemap metadata 21785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // when a program keeps allocating and freeing large blocks. 21795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 21805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (old_system_bytes < kPageMapBigAllocationThreshold 21815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) && system_bytes_ >= kPageMapBigAllocationThreshold) { 21825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pagemap_.PreallocateMoreMemory(); 21835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 21845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 21855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Make sure pagemap_ has entries for all of the new pages. 21865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Plus ensure one before and one after so coalescing code 21875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // does not need bounds-checking. 21885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (pagemap_.Ensure(p-1, ask+2)) { 21895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Pretend the new area is allocated and then Delete() it to 21905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // cause any necessary coalescing to occur. 21915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // 21925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // We do not adjust free_pages_ here since Delete() will do it for us. 21935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Span* span = NewSpan(p, ask); 21945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) RecordSpan(span); 21955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Delete(span); 21965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(Check()); 21975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return true; 21985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } else { 21995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // We could not allocate memory within "pagemap_" 22005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // TODO: Once we can return memory to the system, return the new span 22015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return false; 22025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 22035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 22045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 22055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)bool TCMalloc_PageHeap::Check() { 22065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 22075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t totalFreeCommitted = 0; 22085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 2209926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ASSERT(free_[0].normal.next(entropy_) == &free_[0].normal); 2210926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ASSERT(free_[0].returned.next(entropy_) == &free_[0].returned); 22115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 22125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) totalFreeCommitted = CheckList(&large_.normal, kMaxPages, 1000000000, false); 22135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#else 22145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CheckList(&large_.normal, kMaxPages, 1000000000, false); 22155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 22165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CheckList(&large_.returned, kMaxPages, 1000000000, true); 22175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (Length s = 1; s < kMaxPages; s++) { 22185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 22195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) totalFreeCommitted += CheckList(&free_[s].normal, s, s, false); 22205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#else 22215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CheckList(&free_[s].normal, s, s, false); 22225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 22235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CheckList(&free_[s].returned, s, s, true); 22245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 22255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 22265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(totalFreeCommitted == free_committed_pages_); 22275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 22285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return true; 22295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 22305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 22315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if ASSERT_DISABLED 22325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)size_t TCMalloc_PageHeap::CheckList(Span*, Length, Length, bool) { 22335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 0; 22345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 22355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#else 22365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)size_t TCMalloc_PageHeap::CheckList(Span* list, Length min_pages, Length max_pages, bool decommitted) { 22375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t freeCount = 0; 2238926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) for (Span* s = list->next(entropy_); s != list; s = s->next(entropy_)) { 22395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CHECK_CONDITION(s->free); 22405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CHECK_CONDITION(s->length >= min_pages); 22415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CHECK_CONDITION(s->length <= max_pages); 22425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CHECK_CONDITION(GetDescriptor(s->start) == s); 22435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CHECK_CONDITION(GetDescriptor(s->start+s->length-1) == s); 22445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CHECK_CONDITION(s->decommitted == decommitted); 22455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) freeCount += s->length; 22465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 22475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return freeCount; 22485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 22495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 22505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 22515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void TCMalloc_PageHeap::ReleaseFreeList(Span* list, Span* returned) { 22525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Walk backwards through list so that when we push these 22535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // spans on the "returned" list, we preserve the order. 22545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 22555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t freePageReduction = 0; 22565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 22575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2258926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) while (!DLL_IsEmpty(list, entropy_)) { 2259926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) Span* s = list->prev(entropy_); 22605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2261926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Remove(s, entropy_); 22625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) s->decommitted = true; 2263926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Prepend(returned, s, entropy_); 22645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TCMalloc_SystemRelease(reinterpret_cast<void*>(s->start << kPageShift), 22655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) static_cast<size_t>(s->length << kPageShift)); 22665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 22675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) freePageReduction += s->length; 22685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 22695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 22705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 22715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 22725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) free_committed_pages_ -= freePageReduction; 227302772c6a72f1ee0b226341a4f4439970c29fc861Ben Murdoch if (free_committed_pages_ < min_free_committed_pages_since_last_scavenge_) 22745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) min_free_committed_pages_since_last_scavenge_ = free_committed_pages_; 22755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 22765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 22775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 22785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void TCMalloc_PageHeap::ReleaseFreePages() { 22795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (Length s = 0; s < kMaxPages; s++) { 22805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ReleaseFreeList(&free_[s].normal, &free_[s].returned); 22815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 22825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ReleaseFreeList(&large_.normal, &large_.returned); 22835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(Check()); 22845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 22855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 22865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)//------------------------------------------------------------------- 22875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Free list 22885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)//------------------------------------------------------------------- 22895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 22905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class TCMalloc_ThreadCache_FreeList { 22915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) private: 2292926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL list_; // Linked list of nodes 22935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) uint16_t length_; // Current length 22945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) uint16_t lowater_; // Low water mark for list length 2295926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) uintptr_t entropy_; // Entropy source for hardening 22965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 22975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) public: 2298926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) void Init(uintptr_t entropy) { 2299926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) list_.setValue(NULL); 23005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) length_ = 0; 23015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) lowater_ = 0; 2302926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) entropy_ = entropy; 2303926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#if ENABLE(TCMALLOC_HARDENING) 2304926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ASSERT(entropy_); 2305926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#endif 23065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 23075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 23085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Return current length of list 23095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int length() const { 23105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return length_; 23115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 23125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 23135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Is list empty? 23145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) bool empty() const { 2315926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return !list_; 23165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 23175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 23185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Low-water mark management 23195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int lowwatermark() const { return lowater_; } 23205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void clear_lowwatermark() { lowater_ = length_; } 23215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2322926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ALWAYS_INLINE void Push(HardenedSLL ptr) { 2323926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) SLL_Push(&list_, ptr, entropy_); 23245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) length_++; 23255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 23265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2327926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) void PushRange(int N, HardenedSLL start, HardenedSLL end) { 2328926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) SLL_PushRange(&list_, start, end, entropy_); 23295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) length_ = length_ + static_cast<uint16_t>(N); 23305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 23315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2332926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) void PopRange(int N, HardenedSLL* start, HardenedSLL* end) { 2333926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) SLL_PopRange(&list_, N, start, end, entropy_); 23345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(length_ >= N); 23355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) length_ = length_ - static_cast<uint16_t>(N); 23365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (length_ < lowater_) lowater_ = length_; 23375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 23385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 23395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ALWAYS_INLINE void* Pop() { 2340926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ASSERT(list_); 23415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) length_--; 23425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (length_ < lowater_) lowater_ = length_; 2343926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return SLL_Pop(&list_, entropy_).value(); 2344926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) } 2345926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 2346926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // Runs through the linked list to ensure that 2347926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // we can do that, and ensures that 'missing' 2348926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // is not present 2349926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) NEVER_INLINE void Validate(HardenedSLL missing, size_t size) { 2350926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL node = list_; 2351926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) UNUSED_PARAM(size); 2352926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) while (node) { 2353926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) RELEASE_ASSERT(node != missing); 2354926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) RELEASE_ASSERT(IS_DEFINITELY_POISONED(node.value(), size)); 2355926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) node = SLL_Next(node, entropy_); 2356926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) } 2357926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) } 23585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 23595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) template <class Finder, class Reader> 23605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void enumerateFreeObjects(Finder& finder, const Reader& reader) 23615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) { 2362926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) for (HardenedSLL nextObject = list_; nextObject; nextObject.setValue(reader.nextEntryInHardenedLinkedList(reinterpret_cast<void**>(nextObject.value()), entropy_))) 2363926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) finder.visit(nextObject.value()); 23645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 23655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)}; 23665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 23675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)//------------------------------------------------------------------- 23685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Data kept per thread 23695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)//------------------------------------------------------------------- 23705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 23715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class TCMalloc_ThreadCache { 23725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) private: 23735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) typedef TCMalloc_ThreadCache_FreeList FreeList; 23745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if OS(WINDOWS) 23755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) typedef DWORD ThreadIdentifier; 23765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#else 23775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) typedef pthread_t ThreadIdentifier; 23785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 23795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 23805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t size_; // Combined size of data 23815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ThreadIdentifier tid_; // Which thread owns it 23825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) bool in_setspecific_; // Called pthread_setspecific? 23835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) FreeList list_[kNumClasses]; // Array indexed by size-class 23845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 23855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // We sample allocations, biased by the size of the allocation 23865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) uint32_t rnd_; // Cheap random number generator 23875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t bytes_until_sample_; // Bytes until we sample next 23885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2389926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) uintptr_t entropy_; // Entropy value used for hardening 2390926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 23915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Allocate a new heap. REQUIRES: pageheap_lock is held. 2392926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) static inline TCMalloc_ThreadCache* NewHeap(ThreadIdentifier tid, uintptr_t entropy); 23935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 23945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Use only as pthread thread-specific destructor function. 23955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) static void DestroyThreadCache(void* ptr); 23965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) public: 23975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // All ThreadCache objects are kept in a linked list (for stats collection) 23985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TCMalloc_ThreadCache* next_; 23995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TCMalloc_ThreadCache* prev_; 24005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2401926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) void Init(ThreadIdentifier tid, uintptr_t entropy); 24025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void Cleanup(); 24035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 24045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Accessors (mostly just for printing stats) 24055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int freelist_length(size_t cl) const { return list_[cl].length(); } 24065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 24075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Total byte size in cache 24085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t Size() const { return size_; } 24095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 24105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ALWAYS_INLINE void* Allocate(size_t size); 2411926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) void Deallocate(HardenedSLL ptr, size_t size_class); 24125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 24135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ALWAYS_INLINE void FetchFromCentralCache(size_t cl, size_t allocationSize); 24145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void ReleaseToCentralCache(size_t cl, int N); 24155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void Scavenge(); 24165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void Print() const; 24175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 24185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Record allocation of "k" bytes. Return true iff allocation 24195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // should be sampled 24205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) bool SampleAllocation(size_t k); 24215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 24225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Pick next sampling point 24235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void PickNextSample(size_t k); 24245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 24255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) static void InitModule(); 24265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) static void InitTSD(); 24275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) static TCMalloc_ThreadCache* GetThreadHeap(); 24285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) static TCMalloc_ThreadCache* GetCache(); 24295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) static TCMalloc_ThreadCache* GetCacheIfPresent(); 24305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) static TCMalloc_ThreadCache* CreateCacheIfNecessary(); 24315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) static void DeleteCache(TCMalloc_ThreadCache* heap); 24325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) static void BecomeIdle(); 24335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) static void RecomputeThreadCacheSize(); 24345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 24355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) template <class Finder, class Reader> 24365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void enumerateFreeObjects(Finder& finder, const Reader& reader) 24375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) { 24385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (unsigned sizeClass = 0; sizeClass < kNumClasses; sizeClass++) 24395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) list_[sizeClass].enumerateFreeObjects(finder, reader); 24405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 24415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)}; 24425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 24435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)//------------------------------------------------------------------- 24445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Global variables 24455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)//------------------------------------------------------------------- 24465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 24475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Central cache -- a collection of free-lists, one per size-class. 24485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// We have a separate lock per free-list to reduce contention. 24495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static TCMalloc_Central_FreeListPadded central_cache[kNumClasses]; 24505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 24515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Page-level allocator 24525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static AllocAlignmentInteger pageheap_memory[(sizeof(TCMalloc_PageHeap) + sizeof(AllocAlignmentInteger) - 1) / sizeof(AllocAlignmentInteger)]; 24535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static bool phinited = false; 24545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 24555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Avoid extra level of indirection by making "pageheap" be just an alias 24565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// of pageheap_memory. 24575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)typedef union { 24585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void* m_memory; 24595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TCMalloc_PageHeap* m_pageHeap; 24605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} PageHeapUnion; 24615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 24625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static inline TCMalloc_PageHeap* getPageHeap() 24635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 24645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) PageHeapUnion u = { &pageheap_memory[0] }; 24655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return u.m_pageHeap; 24665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 24675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 24685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#define pageheap getPageHeap() 24695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2470926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)size_t fastMallocGoodSize(size_t bytes) 2471926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles){ 2472926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if (!phinited) 2473926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) TCMalloc_ThreadCache::InitModule(); 2474926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return AllocationSize(bytes); 2475926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)} 2476926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 24775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 24785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 24795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if HAVE(DISPATCH_H) || OS(WINDOWS) 24805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 24815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void TCMalloc_PageHeap::periodicScavenge() 24825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 24835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) SpinLockHolder h(&pageheap_lock); 24845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pageheap->scavenge(); 24855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 24865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (shouldScavenge()) { 24875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) rescheduleScavenger(); 24885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return; 24895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 24905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 24915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) suspendScavenger(); 24925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 24935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 24945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)ALWAYS_INLINE void TCMalloc_PageHeap::signalScavenger() 24955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 24965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(pageheap_lock.IsHeld()); 24975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (isScavengerSuspended() && shouldScavenge()) 24985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) scheduleScavenger(); 24995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 25005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 25015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#else 25025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 25035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void TCMalloc_PageHeap::scavengerThread() 25045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 25055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if HAVE(PTHREAD_SETNAME_NP) 25065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_setname_np("JavaScriptCore: FastMalloc scavenger"); 25075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 25085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 25095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) while (1) { 25105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pageheap_lock.Lock(); 25115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (!shouldScavenge()) { 25125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Set to false so that signalScavenger() will check whether we need to be siganlled. 25135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_scavengeThreadActive = false; 25145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 25155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // We need to unlock now, as this thread will block on the condvar until scavenging is required. 25165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pageheap_lock.Unlock(); 25175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 25185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Block until there are enough free committed pages to release back to the system. 25195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_mutex_lock(&m_scavengeMutex); 25205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_cond_wait(&m_scavengeCondition, &m_scavengeMutex); 25215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // After exiting the pthread_cond_wait, we hold the lock on m_scavengeMutex. Unlock it to prevent 25225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // deadlock next time round the loop. 25235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_mutex_unlock(&m_scavengeMutex); 25245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 25255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Set to true to prevent unnecessary signalling of the condvar. 25265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_scavengeThreadActive = true; 25275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } else 25285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pageheap_lock.Unlock(); 25295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 25305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Wait for a while to calculate how much memory remains unused during this pause. 25315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) sleep(kScavengeDelayInSeconds); 25325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 25335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) { 25345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) SpinLockHolder h(&pageheap_lock); 25355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pageheap->scavenge(); 25365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 25375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 25385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 25395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 25405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 25415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 25425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 25435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 25445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// If TLS is available, we also store a copy 25455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// of the per-thread object in a __thread variable 25465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// since __thread variables are faster to read 25475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// than pthread_getspecific(). We still need 25485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// pthread_setspecific() because __thread 25495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// variables provide no way to run cleanup 25505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// code when a thread is destroyed. 25515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#ifdef HAVE_TLS 25525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static __thread TCMalloc_ThreadCache *threadlocal_heap; 25535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 25545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Thread-specific key. Initialization here is somewhat tricky 25555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// because some Linux startup code invokes malloc() before it 25565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// is in a good enough state to handle pthread_keycreate(). 25575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Therefore, we use TSD keys only after tsd_inited is set to true. 25585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Until then, we use a slow path to get the heap object. 25595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static bool tsd_inited = false; 25605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static pthread_key_t heap_key; 25615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if OS(WINDOWS) 25625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)DWORD tlsIndex = TLS_OUT_OF_INDEXES; 25635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 25645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 25655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static ALWAYS_INLINE void setThreadHeap(TCMalloc_ThreadCache* heap) 25665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 25675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if USE(PTHREAD_GETSPECIFIC_DIRECT) 25685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Can't have two libraries both doing this in the same process, 25695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // so check and make this crash right away. 25705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (pthread_getspecific(heap_key)) 25715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CRASH(); 25725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 25735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 25745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Still do pthread_setspecific even if there's an alternate form 25755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // of thread-local storage in use, to benefit from the delete callback. 25765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_setspecific(heap_key, heap); 25775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 25785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if OS(WINDOWS) 25795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TlsSetValue(tlsIndex, heap); 25805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 25815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 25825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 25835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Allocator for thread heaps 25845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static PageHeapAllocator<TCMalloc_ThreadCache> threadheap_allocator; 25855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 25865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Linked list of heap objects. Protected by pageheap_lock. 25875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static TCMalloc_ThreadCache* thread_heaps = NULL; 25885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static int thread_heap_count = 0; 25895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 25905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Overall thread cache size. Protected by pageheap_lock. 25915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static size_t overall_thread_cache_size = kDefaultOverallThreadCacheSize; 25925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 25935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Global per-thread cache size. Writes are protected by 25945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// pageheap_lock. Reads are done without any locking, which should be 25955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// fine as long as size_t can be written atomically and we don't place 25965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// invariants between this variable and other pieces of state. 25975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static volatile size_t per_thread_cache_size = kMaxThreadCacheSize; 25985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 25995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)//------------------------------------------------------------------- 26005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Central cache implementation 26015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)//------------------------------------------------------------------- 26025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2603926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)void TCMalloc_Central_FreeList::Init(size_t cl, uintptr_t entropy) { 26045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) lock_.Init(); 26055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_class_ = cl; 2606926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) entropy_ = entropy; 2607926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#if ENABLE(TCMALLOC_HARDENING) 2608926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ASSERT(entropy_); 2609926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#endif 2610926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Init(&empty_, entropy_); 2611926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Init(&nonempty_, entropy_); 26125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) counter_ = 0; 26135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 26145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) cache_size_ = 1; 26155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) used_slots_ = 0; 26165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(cache_size_ <= kNumTransferEntries); 26175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 26185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2619926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)void TCMalloc_Central_FreeList::ReleaseListToSpans(HardenedSLL start) { 26205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) while (start) { 2621926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL next = SLL_Next(start, entropy_); 26225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ReleaseToSpans(start); 26235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) start = next; 26245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 26255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 26265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2627926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)ALWAYS_INLINE void TCMalloc_Central_FreeList::ReleaseToSpans(HardenedSLL object) { 2628926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) const PageID p = reinterpret_cast<uintptr_t>(object.value()) >> kPageShift; 26295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Span* span = pageheap->GetDescriptor(p); 26305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(span != NULL); 26315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(span->refcount > 0); 26325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 26335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // If span is empty, move it to non-empty list 2634926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if (!span->objects) { 2635926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Remove(span, entropy_); 2636926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Prepend(&nonempty_, span, entropy_); 26375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Event(span, 'N', 0); 26385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 26395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 26405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // The following check is expensive, so it is disabled by default 26415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (false) { 26425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Check that object does not occur in list 26435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) unsigned got = 0; 2644926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) for (HardenedSLL p = span->objects; !p; SLL_Next(p, entropy_)) { 2645926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ASSERT(p.value() != object.value()); 26465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) got++; 26475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 26485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(got + span->refcount == 26495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) (span->length<<kPageShift)/ByteSizeForClass(span->sizeclass)); 26505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 26515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 26525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) counter_++; 26535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) span->refcount--; 26545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (span->refcount == 0) { 26555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Event(span, '#', 0); 26565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) counter_ -= (span->length<<kPageShift) / ByteSizeForClass(span->sizeclass); 2657926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Remove(span, entropy_); 26585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 26595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Release central list lock while operating on pageheap 26605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) lock_.Unlock(); 26615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) { 26625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) SpinLockHolder h(&pageheap_lock); 26635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pageheap->Delete(span); 26645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 26655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) lock_.Lock(); 26665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } else { 2667926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) SLL_SetNext(object, span->objects, entropy_); 2668926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) span->objects.setValue(object.value()); 26695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 26705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 26715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 26725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)ALWAYS_INLINE bool TCMalloc_Central_FreeList::EvictRandomSizeClass( 26735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t locked_size_class, bool force) { 26745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) static int race_counter = 0; 26755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int t = race_counter++; // Updated without a lock, but who cares. 26765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (t >= static_cast<int>(kNumClasses)) { 26775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) while (t >= static_cast<int>(kNumClasses)) { 26785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) t -= kNumClasses; 26795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 26805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) race_counter = t; 26815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 26825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(t >= 0); 26835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(t < static_cast<int>(kNumClasses)); 26845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (t == static_cast<int>(locked_size_class)) return false; 26855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return central_cache[t].ShrinkCache(static_cast<int>(locked_size_class), force); 26865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 26875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 26885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)bool TCMalloc_Central_FreeList::MakeCacheSpace() { 26895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Is there room in the cache? 26905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (used_slots_ < cache_size_) return true; 26915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Check if we can expand this cache? 26925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (cache_size_ == kNumTransferEntries) return false; 26935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Ok, we'll try to grab an entry from some other size class. 26945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (EvictRandomSizeClass(size_class_, false) || 26955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) EvictRandomSizeClass(size_class_, true)) { 26965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Succeeded in evicting, we're going to make our cache larger. 26975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) cache_size_++; 26985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return true; 26995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 27005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return false; 27015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 27025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 27035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 27045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)namespace { 27055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class LockInverter { 27065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) private: 27075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) SpinLock *held_, *temp_; 27085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) public: 27095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) inline explicit LockInverter(SpinLock* held, SpinLock *temp) 27105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) : held_(held), temp_(temp) { held_->Unlock(); temp_->Lock(); } 27115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) inline ~LockInverter() { temp_->Unlock(); held_->Lock(); } 27125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)}; 27135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 27145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 27155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)bool TCMalloc_Central_FreeList::ShrinkCache(int locked_size_class, bool force) { 27165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Start with a quick check without taking a lock. 27175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (cache_size_ == 0) return false; 27185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // We don't evict from a full cache unless we are 'forcing'. 27195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (force == false && used_slots_ == cache_size_) return false; 27205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 27215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Grab lock, but first release the other lock held by this thread. We use 27225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // the lock inverter to ensure that we never hold two size class locks 27235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // concurrently. That can create a deadlock because there is no well 27245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // defined nesting order. 27255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) LockInverter li(¢ral_cache[locked_size_class].lock_, &lock_); 27265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(used_slots_ <= cache_size_); 27275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(0 <= cache_size_); 27285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (cache_size_ == 0) return false; 27295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (used_slots_ == cache_size_) { 27305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (force == false) return false; 27315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // ReleaseListToSpans releases the lock, so we have to make all the 27325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // updates to the central list before calling it. 27335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) cache_size_--; 27345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) used_slots_--; 27355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ReleaseListToSpans(tc_slots_[used_slots_].head); 27365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return true; 27375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 27385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) cache_size_--; 27395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return true; 27405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 27415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2742926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)void TCMalloc_Central_FreeList::InsertRange(HardenedSLL start, HardenedSLL end, int N) { 27435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) SpinLockHolder h(&lock_); 27445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (N == num_objects_to_move[size_class_] && 27455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) MakeCacheSpace()) { 27465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int slot = used_slots_++; 27475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(slot >=0); 27485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(slot < kNumTransferEntries); 27495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TCEntry *entry = &tc_slots_[slot]; 27505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) entry->head = start; 27515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) entry->tail = end; 27525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return; 27535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 27545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ReleaseListToSpans(start); 27555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 27565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2757926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)void TCMalloc_Central_FreeList::RemoveRange(HardenedSLL* start, HardenedSLL* end, int *N) { 27585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int num = *N; 27595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(num > 0); 27605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 27615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) SpinLockHolder h(&lock_); 27625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (num == num_objects_to_move[size_class_] && used_slots_ > 0) { 27635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int slot = --used_slots_; 27645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(slot >= 0); 27655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TCEntry *entry = &tc_slots_[slot]; 27665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) *start = entry->head; 27675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) *end = entry->tail; 27685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return; 27695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 27705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 27715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // TODO: Prefetch multiple TCEntries? 2772926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL tail = FetchFromSpansSafe(); 27735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (!tail) { 27745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // We are completely out of memory. 2775926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) *start = *end = HardenedSLL::null(); 27765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) *N = 0; 27775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return; 27785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 27795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2780926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) SLL_SetNext(tail, HardenedSLL::null(), entropy_); 2781926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL head = tail; 27825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int count = 1; 27835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) while (count < num) { 2784926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL t = FetchFromSpans(); 27855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (!t) break; 2786926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) SLL_Push(&head, t, entropy_); 27875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) count++; 27885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 27895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) *start = head; 27905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) *end = tail; 27915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) *N = count; 27925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 27935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 27945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2795926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)HardenedSLL TCMalloc_Central_FreeList::FetchFromSpansSafe() { 2796926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL t = FetchFromSpans(); 27975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (!t) { 27985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Populate(); 27995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) t = FetchFromSpans(); 28005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 28015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return t; 28025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 28035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2804926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)HardenedSLL TCMalloc_Central_FreeList::FetchFromSpans() { 2805926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if (DLL_IsEmpty(&nonempty_, entropy_)) return HardenedSLL::null(); 2806926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) Span* span = nonempty_.next(entropy_); 28075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2808926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ASSERT(span->objects); 28095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT_SPAN_COMMITTED(span); 28105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) span->refcount++; 2811926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL result = span->objects; 2812926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) span->objects = SLL_Next(result, entropy_); 2813926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if (!span->objects) { 28145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Move to empty list 2815926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Remove(span, entropy_); 2816926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Prepend(&empty_, span, entropy_); 28175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Event(span, 'E', 0); 28185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 28195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) counter_--; 28205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return result; 28215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 28225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 28235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Fetch memory from the system and add to the central cache freelist. 28245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)ALWAYS_INLINE void TCMalloc_Central_FreeList::Populate() { 28255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Release central list lock while operating on pageheap 28265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) lock_.Unlock(); 28275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const size_t npages = class_to_pages[size_class_]; 28285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 28295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Span* span; 28305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) { 28315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) SpinLockHolder h(&pageheap_lock); 28325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) span = pageheap->New(npages); 28335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (span) pageheap->RegisterSizeClass(span, size_class_); 28345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 28355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (span == NULL) { 2836f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles)#if OS(WINDOWS) 28375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) MESSAGE("allocation failed: %d\n", ::GetLastError()); 28385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#else 2839f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) MESSAGE("allocation failed: %d\n", errno); 28405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 28415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) lock_.Lock(); 28425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return; 28435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 28445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT_SPAN_COMMITTED(span); 28455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(span->length == npages); 28465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Cache sizeclass info eagerly. Locking is not necessary. 28475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // (Instead of being eager, we could just replace any stale info 28485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // about this span, but that seems to be no better in practice.) 28495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (size_t i = 0; i < npages; i++) { 28505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pageheap->CacheSizeClass(span->start + i, size_class_); 28515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 28525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 28535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Split the block into pieces and add to the free-list 28545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // TODO: coloring of objects to avoid cache conflicts? 2855926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL head = HardenedSLL::null(); 2856926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) char* start = reinterpret_cast<char*>(span->start << kPageShift); 28575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const size_t size = ByteSizeForClass(size_class_); 2858926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) char* ptr = start + (npages << kPageShift) - ((npages << kPageShift) % size); 28595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int num = 0; 2860926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#if ENABLE(TCMALLOC_HARDENING) 2861926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) uint32_t startPoison = freedObjectStartPoison(); 2862926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) uint32_t endPoison = freedObjectEndPoison(); 2863926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#endif 2864926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 2865926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) while (ptr > start) { 2866926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ptr -= size; 2867926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL node = HardenedSLL::create(ptr); 2868926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) POISON_DEALLOCATION_EXPLICIT(ptr, size, startPoison, endPoison); 2869926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) SLL_SetNext(node, head, entropy_); 2870926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) head = node; 28715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) num++; 28725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 2873926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ASSERT(ptr == start); 2874926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ASSERT(ptr == head.value()); 2875926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#ifndef NDEBUG 2876926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) { 2877926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL node = head; 2878926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) while (node) { 2879926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ASSERT(IS_DEFINITELY_POISONED(node.value(), size)); 2880926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) node = SLL_Next(node, entropy_); 2881926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) } 2882926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) } 2883926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#endif 2884926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) span->objects = head; 2885926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ASSERT(span->objects.value() == head.value()); 28865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) span->refcount = 0; // No sub-object in use yet 28875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 28885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Add span to list of non-empty spans 28895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) lock_.Lock(); 2890926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Prepend(&nonempty_, span, entropy_); 28915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) counter_ += num; 28925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 28935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 28945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)//------------------------------------------------------------------- 28955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// TCMalloc_ThreadCache implementation 28965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)//------------------------------------------------------------------- 28975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 28985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)inline bool TCMalloc_ThreadCache::SampleAllocation(size_t k) { 28995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (bytes_until_sample_ < k) { 29005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) PickNextSample(k); 29015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return true; 29025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } else { 29035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) bytes_until_sample_ -= k; 29045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return false; 29055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 29065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 29075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2908926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)void TCMalloc_ThreadCache::Init(ThreadIdentifier tid, uintptr_t entropy) { 29095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_ = 0; 29105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) next_ = NULL; 29115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) prev_ = NULL; 29125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tid_ = tid; 29135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) in_setspecific_ = false; 2914926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) entropy_ = entropy; 2915926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#if ENABLE(TCMALLOC_HARDENING) 2916926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ASSERT(entropy_); 2917926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#endif 29185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (size_t cl = 0; cl < kNumClasses; ++cl) { 2919926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) list_[cl].Init(entropy_); 29205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 29215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 29225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Initialize RNG -- run it for a bit to get to good values 29235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) bytes_until_sample_ = 0; 29245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) rnd_ = static_cast<uint32_t>(reinterpret_cast<uintptr_t>(this)); 29255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (int i = 0; i < 100; i++) { 29265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) PickNextSample(static_cast<size_t>(FLAGS_tcmalloc_sample_parameter * 2)); 29275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 29285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 29295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 29305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void TCMalloc_ThreadCache::Cleanup() { 29315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Put unused memory back into central cache 29325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (size_t cl = 0; cl < kNumClasses; ++cl) { 29335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (list_[cl].length() > 0) { 29345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ReleaseToCentralCache(cl, list_[cl].length()); 29355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 29365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 29375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 29385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 29395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)ALWAYS_INLINE void* TCMalloc_ThreadCache::Allocate(size_t size) { 29405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(size <= kMaxSize); 29415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const size_t cl = SizeClass(size); 29425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) FreeList* list = &list_[cl]; 29435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t allocationSize = ByteSizeForClass(cl); 29445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (list->empty()) { 29455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) FetchFromCentralCache(cl, allocationSize); 29465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (list->empty()) return NULL; 29475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 29485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_ -= allocationSize; 2949926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) void* result = list->Pop(); 2950926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if (!result) 2951926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return 0; 2952926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) RELEASE_ASSERT(IS_DEFINITELY_POISONED(result, allocationSize)); 2953926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) POISON_ALLOCATION(result, allocationSize); 2954926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return result; 29555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 29565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2957926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)inline void TCMalloc_ThreadCache::Deallocate(HardenedSLL ptr, size_t cl) { 2958926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) size_t allocationSize = ByteSizeForClass(cl); 2959926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) size_ += allocationSize; 29605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) FreeList* list = &list_[cl]; 2961926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if (MAY_BE_POISONED(ptr.value(), allocationSize)) 2962926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) list->Validate(ptr, allocationSize); 2963926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 2964926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) POISON_DEALLOCATION(ptr.value(), allocationSize); 29655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) list->Push(ptr); 29665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // If enough data is free, put back into central cache 29675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (list->length() > kMaxFreeListLength) { 29685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ReleaseToCentralCache(cl, num_objects_to_move[cl]); 29695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 29705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (size_ >= per_thread_cache_size) Scavenge(); 29715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 29725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 29735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Remove some objects of class "cl" from central cache and add to thread heap 29745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)ALWAYS_INLINE void TCMalloc_ThreadCache::FetchFromCentralCache(size_t cl, size_t allocationSize) { 29755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int fetch_count = num_objects_to_move[cl]; 2976926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL start, end; 29775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) central_cache[cl].RemoveRange(&start, &end, &fetch_count); 29785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) list_[cl].PushRange(fetch_count, start, end); 29795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_ += allocationSize * fetch_count; 29805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 29815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 29825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Remove some objects of class "cl" from thread heap and add to central cache 29835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)inline void TCMalloc_ThreadCache::ReleaseToCentralCache(size_t cl, int N) { 29845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(N > 0); 29855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) FreeList* src = &list_[cl]; 29865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (N > src->length()) N = src->length(); 29875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_ -= N*ByteSizeForClass(cl); 29885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 29895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // We return prepackaged chains of the correct size to the central cache. 29905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // TODO: Use the same format internally in the thread caches? 29915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int batch_size = num_objects_to_move[cl]; 29925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) while (N > batch_size) { 2993926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL tail, head; 29945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) src->PopRange(batch_size, &head, &tail); 29955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) central_cache[cl].InsertRange(head, tail, batch_size); 29965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) N -= batch_size; 29975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 2998926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) HardenedSLL tail, head; 29995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) src->PopRange(N, &head, &tail); 30005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) central_cache[cl].InsertRange(head, tail, N); 30015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 30025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 30035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Release idle memory to the central cache 30045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)inline void TCMalloc_ThreadCache::Scavenge() { 30055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // If the low-water mark for the free list is L, it means we would 30065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // not have had to allocate anything from the central cache even if 30075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // we had reduced the free list size by L. We aim to get closer to 30085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // that situation by dropping L/2 nodes from the free list. This 30095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // may not release much memory, but if so we will call scavenge again 30105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // pretty soon and the low-water marks will be high on that call. 30115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) //int64 start = CycleClock::Now(); 30125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 30135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (size_t cl = 0; cl < kNumClasses; cl++) { 30145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) FreeList* list = &list_[cl]; 30155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const int lowmark = list->lowwatermark(); 30165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (lowmark > 0) { 30175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const int drop = (lowmark > 1) ? lowmark/2 : 1; 30185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ReleaseToCentralCache(cl, drop); 30195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 30205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) list->clear_lowwatermark(); 30215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 30225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 30235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) //int64 finish = CycleClock::Now(); 30245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) //CycleTimer ct; 30255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) //MESSAGE("GC: %.0f ns\n", ct.CyclesToUsec(finish-start)*1000.0); 30265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 30275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 30285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void TCMalloc_ThreadCache::PickNextSample(size_t k) { 30295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Make next "random" number 30305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // x^32+x^22+x^2+x^1+1 is a primitive polynomial for random numbers 30315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) static const uint32_t kPoly = (1 << 22) | (1 << 2) | (1 << 1) | (1 << 0); 30325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) uint32_t r = rnd_; 30335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) rnd_ = (r << 1) ^ ((static_cast<int32_t>(r) >> 31) & kPoly); 30345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 30355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Next point is "rnd_ % (sample_period)". I.e., average 30365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // increment is "sample_period/2". 30375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const int flag_value = static_cast<int>(FLAGS_tcmalloc_sample_parameter); 30385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) static int last_flag_value = -1; 30395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 30405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (flag_value != last_flag_value) { 30415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) SpinLockHolder h(&sample_period_lock); 30425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int i; 30435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (i = 0; i < (static_cast<int>(sizeof(primes_list)/sizeof(primes_list[0])) - 1); i++) { 30445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (primes_list[i] >= flag_value) { 30455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) break; 30465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 30475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 30485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) sample_period = primes_list[i]; 30495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) last_flag_value = flag_value; 30505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 30515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 30525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) bytes_until_sample_ += rnd_ % sample_period; 30535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 30545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (k > (static_cast<size_t>(-1) >> 2)) { 30555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // If the user has asked for a huge allocation then it is possible 30565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // for the code below to loop infinitely. Just return (note that 30575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // this throws off the sampling accuracy somewhat, but a user who 30585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // is allocating more than 1G of memory at a time can live with a 30595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // minor inaccuracy in profiling of small allocations, and also 30605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // would rather not wait for the loop below to terminate). 30615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return; 30625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 30635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 30645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) while (bytes_until_sample_ < k) { 30655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Increase bytes_until_sample_ by enough average sampling periods 30665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // (sample_period >> 1) to allow us to sample past the current 30675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // allocation. 30685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) bytes_until_sample_ += (sample_period >> 1); 30695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 30705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 30715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) bytes_until_sample_ -= k; 30725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 30735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 30745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void TCMalloc_ThreadCache::InitModule() { 30755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // There is a slight potential race here because of double-checked 30765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // locking idiom. However, as long as the program does a small 30775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // allocation before switching to multi-threaded mode, we will be 30785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // fine. We increase the chances of doing such a small allocation 30795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // by doing one in the constructor of the module_enter_exit_hook 30805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // object declared below. 30815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) SpinLockHolder h(&pageheap_lock); 30825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (!phinited) { 3083926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) uintptr_t entropy = HARDENING_ENTROPY; 30845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) InitTSD(); 30855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) InitSizeClasses(); 3086926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) threadheap_allocator.Init(entropy); 3087926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) span_allocator.Init(entropy); 30885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) span_allocator.New(); // Reduce cache conflicts 30895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) span_allocator.New(); // Reduce cache conflicts 3090926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) stacktrace_allocator.Init(entropy); 3091926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) DLL_Init(&sampled_objects, entropy); 30925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (size_t i = 0; i < kNumClasses; ++i) { 3093926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) central_cache[i].Init(i, entropy); 30945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 30955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pageheap->init(); 30965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) phinited = 1; 309753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)#if OS(DARWIN) 30985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) FastMallocZone::init(); 30995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 31005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 31015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 31025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3103926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)inline TCMalloc_ThreadCache* TCMalloc_ThreadCache::NewHeap(ThreadIdentifier tid, uintptr_t entropy) { 31045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Create the heap and add it to the linked list 31055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TCMalloc_ThreadCache *heap = threadheap_allocator.New(); 3106926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) heap->Init(tid, entropy); 31075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) heap->next_ = thread_heaps; 31085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) heap->prev_ = NULL; 31095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (thread_heaps != NULL) thread_heaps->prev_ = heap; 31105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) thread_heaps = heap; 31115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) thread_heap_count++; 31125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) RecomputeThreadCacheSize(); 31135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return heap; 31145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 31155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 31165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)inline TCMalloc_ThreadCache* TCMalloc_ThreadCache::GetThreadHeap() { 31175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#ifdef HAVE_TLS 31185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // __thread is faster, but only when the kernel supports it 31195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (KernelSupportsTLS()) 31205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return threadlocal_heap; 31215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#elif OS(WINDOWS) 31225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return static_cast<TCMalloc_ThreadCache*>(TlsGetValue(tlsIndex)); 31235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#else 31245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return static_cast<TCMalloc_ThreadCache*>(pthread_getspecific(heap_key)); 31255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 31265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 31275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 31285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)inline TCMalloc_ThreadCache* TCMalloc_ThreadCache::GetCache() { 31295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TCMalloc_ThreadCache* ptr = NULL; 31305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (!tsd_inited) { 31315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) InitModule(); 31325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } else { 31335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ptr = GetThreadHeap(); 31345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 31355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (ptr == NULL) ptr = CreateCacheIfNecessary(); 31365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return ptr; 31375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 31385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 31395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// In deletion paths, we do not try to create a thread-cache. This is 31405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// because we may be in the thread destruction code and may have 31415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// already cleaned up the cache for this thread. 31425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)inline TCMalloc_ThreadCache* TCMalloc_ThreadCache::GetCacheIfPresent() { 31435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (!tsd_inited) return NULL; 31445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void* const p = GetThreadHeap(); 31455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return reinterpret_cast<TCMalloc_ThreadCache*>(p); 31465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 31475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 31485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void TCMalloc_ThreadCache::InitTSD() { 31495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(!tsd_inited); 31505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if USE(PTHREAD_GETSPECIFIC_DIRECT) 31515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_key_init_np(heap_key, DestroyThreadCache); 31525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#else 31535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_key_create(&heap_key, DestroyThreadCache); 31545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 31555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if OS(WINDOWS) 31565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tlsIndex = TlsAlloc(); 31575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 31585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tsd_inited = true; 315902772c6a72f1ee0b226341a4f4439970c29fc861Ben Murdoch 31605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if !OS(WINDOWS) 31615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // We may have used a fake pthread_t for the main thread. Fix it. 31625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_t zero; 31635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) memset(&zero, 0, sizeof(zero)); 31645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 31655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(pageheap_lock.IsHeld()); 31665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (TCMalloc_ThreadCache* h = thread_heaps; h != NULL; h = h->next_) { 31675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if OS(WINDOWS) 31685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (h->tid_ == 0) { 31695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) h->tid_ = GetCurrentThreadId(); 31705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 31715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#else 31725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (pthread_equal(h->tid_, zero)) { 31735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) h->tid_ = pthread_self(); 31745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 31755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 31765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 31775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 31785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 31795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)TCMalloc_ThreadCache* TCMalloc_ThreadCache::CreateCacheIfNecessary() { 31805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Initialize per-thread data if necessary 31815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TCMalloc_ThreadCache* heap = NULL; 31825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) { 31835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) SpinLockHolder h(&pageheap_lock); 31845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 31855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if OS(WINDOWS) 31865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) DWORD me; 31875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (!tsd_inited) { 31885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) me = 0; 31895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } else { 31905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) me = GetCurrentThreadId(); 31915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 31925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#else 31935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Early on in glibc's life, we cannot even call pthread_self() 31945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pthread_t me; 31955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (!tsd_inited) { 31965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) memset(&me, 0, sizeof(me)); 31975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } else { 31985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) me = pthread_self(); 31995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 32005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 32015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 32025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // This may be a recursive malloc call from pthread_setspecific() 32035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // In that case, the heap for this thread has already been created 32045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // and added to the linked list. So we search for that first. 32055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (TCMalloc_ThreadCache* h = thread_heaps; h != NULL; h = h->next_) { 32065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if OS(WINDOWS) 32075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (h->tid_ == me) { 32085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#else 32095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (pthread_equal(h->tid_, me)) { 32105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 32115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) heap = h; 32125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) break; 32135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 32145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 32155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3216926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if (heap == NULL) heap = NewHeap(me, HARDENING_ENTROPY); 32175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 32185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 32195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // We call pthread_setspecific() outside the lock because it may 32205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // call malloc() recursively. The recursive call will never get 32215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // here again because it will find the already allocated heap in the 32225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // linked list of heaps. 32235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (!heap->in_setspecific_ && tsd_inited) { 32245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) heap->in_setspecific_ = true; 32255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) setThreadHeap(heap); 32265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 32275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return heap; 32285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 32295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 32305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void TCMalloc_ThreadCache::BecomeIdle() { 32315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (!tsd_inited) return; // No caches yet 32325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TCMalloc_ThreadCache* heap = GetThreadHeap(); 32335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (heap == NULL) return; // No thread cache to remove 32345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (heap->in_setspecific_) return; // Do not disturb the active caller 32355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 32365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) heap->in_setspecific_ = true; 32375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) setThreadHeap(NULL); 32385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#ifdef HAVE_TLS 32395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Also update the copy in __thread 32405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) threadlocal_heap = NULL; 32415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 32425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) heap->in_setspecific_ = false; 32435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (GetThreadHeap() == heap) { 32445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Somehow heap got reinstated by a recursive call to malloc 32455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // from pthread_setspecific. We give up in this case. 32465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return; 32475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 32485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 32495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // We can now get rid of the heap 32505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) DeleteCache(heap); 32515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 32525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 32535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void TCMalloc_ThreadCache::DestroyThreadCache(void* ptr) { 32545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Note that "ptr" cannot be NULL since pthread promises not 32555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // to invoke the destructor on NULL values, but for safety, 32565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // we check anyway. 32575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (ptr == NULL) return; 32585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#ifdef HAVE_TLS 32595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Prevent fast path of GetThreadHeap() from returning heap. 32605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) threadlocal_heap = NULL; 32615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 32625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) DeleteCache(reinterpret_cast<TCMalloc_ThreadCache*>(ptr)); 32635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 32645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 32655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void TCMalloc_ThreadCache::DeleteCache(TCMalloc_ThreadCache* heap) { 32665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Remove all memory from heap 32675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) heap->Cleanup(); 32685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 32695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Remove from linked list 32705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) SpinLockHolder h(&pageheap_lock); 32715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (heap->next_ != NULL) heap->next_->prev_ = heap->prev_; 32725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (heap->prev_ != NULL) heap->prev_->next_ = heap->next_; 32735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (thread_heaps == heap) thread_heaps = heap->next_; 32745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) thread_heap_count--; 32755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) RecomputeThreadCacheSize(); 32765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 32775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) threadheap_allocator.Delete(heap); 32785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 32795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 32805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void TCMalloc_ThreadCache::RecomputeThreadCacheSize() { 32815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Divide available space across threads 32825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int n = thread_heap_count > 0 ? thread_heap_count : 1; 32835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t space = overall_thread_cache_size / n; 32845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 32855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Limit to allowed range 32865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (space < kMinThreadCacheSize) space = kMinThreadCacheSize; 32875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (space > kMaxThreadCacheSize) space = kMaxThreadCacheSize; 32885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 32895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) per_thread_cache_size = space; 32905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 32915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 32925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void TCMalloc_ThreadCache::Print() const { 32935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (size_t cl = 0; cl < kNumClasses; ++cl) { 32945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) MESSAGE(" %5" PRIuS " : %4d len; %4d lo\n", 32955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ByteSizeForClass(cl), 32965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) list_[cl].length(), 32975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) list_[cl].lowwatermark()); 32985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 32995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 33005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 33015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Extract interesting stats 33025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)struct TCMallocStats { 33035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) uint64_t system_bytes; // Bytes alloced from system 33045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) uint64_t thread_bytes; // Bytes in thread caches 33055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) uint64_t central_bytes; // Bytes in central cache 33065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) uint64_t transfer_bytes; // Bytes in central transfer cache 33075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) uint64_t pageheap_bytes; // Bytes in page heap 33085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) uint64_t metadata_bytes; // Bytes alloced for metadata 33095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)}; 33105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 33115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// The constructor allocates an object to ensure that initialization 33125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// runs before main(), and therefore we do not have a chance to become 33135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// multi-threaded before initialization. We also create the TSD key 33145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// here. Presumably by the time this constructor runs, glibc is in 33155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// good enough shape to handle pthread_key_create(). 33165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 33175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// The constructor also takes the opportunity to tell STL to use 33185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// tcmalloc. We want to do this early, before construct time, so 33195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// all user STL allocations go through tcmalloc (which works really 33205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// well for STL). 33215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 33225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// The destructor prints stats when the program exits. 33235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class TCMallocGuard { 33245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) public: 33255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 33265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TCMallocGuard() { 33275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#ifdef HAVE_TLS // this is true if the cc/ld/libc combo support TLS 33285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Check whether the kernel also supports TLS (needs to happen at runtime) 33295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CheckIfKernelSupportsTLS(); 33305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 33315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) free(malloc(1)); 33325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TCMalloc_ThreadCache::InitTSD(); 33335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) free(malloc(1)); 33345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 33355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)}; 33365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 33375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)//------------------------------------------------------------------- 33385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Helpers for the exported routines below 33395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)//------------------------------------------------------------------- 33405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 33415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static inline bool CheckCachedSizeClass(void *ptr) { 33425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; 33435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t cached_value = pageheap->GetSizeClassIfCached(p); 33445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return cached_value == 0 || 33455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) cached_value == pageheap->GetDescriptor(p)->sizeclass; 33465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 33475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 33485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static inline void* CheckedMallocResult(void *result) 33495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 33505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(result == 0 || CheckCachedSizeClass(result)); 33515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return result; 33525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 33535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 33545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static inline void* SpanToMallocResult(Span *span) { 33555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT_SPAN_COMMITTED(span); 33565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pageheap->CacheSizeClass(span->start, 0); 3357926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) void* result = reinterpret_cast<void*>(span->start << kPageShift); 3358926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) POISON_ALLOCATION(result, span->length << kPageShift); 3359926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return CheckedMallocResult(result); 33605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 33615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 33625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static ALWAYS_INLINE void* do_malloc(size_t size) { 3363f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) void* ret = 0; 33645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 33655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(!isForbidden()); 33665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3367f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) // The following call forces module initialization 3368f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) TCMalloc_ThreadCache* heap = TCMalloc_ThreadCache::GetCache(); 3369f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) if (size > kMaxSize) { 3370f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) // Use page-level allocator 3371f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) SpinLockHolder h(&pageheap_lock); 3372f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) Span* span = pageheap->New(pages(size)); 3373f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) if (span) 3374f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) ret = SpanToMallocResult(span); 3375f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) } else { 3376f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) // The common case, and also the simplest. This just pops the 3377f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) // size-appropriate freelist, afer replenishing it if it's empty. 3378f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) ret = CheckedMallocResult(heap->Allocate(size)); 33795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 3380f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) if (!ret) 33815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CRASH(); 3382f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) return ret; 33835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 33845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 33855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static ALWAYS_INLINE void do_free(void* ptr) { 33865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (ptr == NULL) return; 33875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(pageheap != NULL); // Should not call free() before malloc() 33885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; 33895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Span* span = NULL; 33905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t cl = pageheap->GetSizeClassIfCached(p); 33915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 33925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (cl == 0) { 33935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) span = pageheap->GetDescriptor(p); 3394926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) RELEASE_ASSERT(span->isValid()); 33955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) cl = span->sizeclass; 33965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pageheap->CacheSizeClass(p, cl); 33975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 33985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (cl != 0) { 33995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#ifndef NO_TCMALLOC_SAMPLES 34005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(!pageheap->GetDescriptor(p)->sample); 34015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 34025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TCMalloc_ThreadCache* heap = TCMalloc_ThreadCache::GetCacheIfPresent(); 34035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (heap != NULL) { 3404926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) heap->Deallocate(HardenedSLL::create(ptr), cl); 34055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } else { 34065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Delete directly into central cache 3407926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) POISON_DEALLOCATION(ptr, ByteSizeForClass(cl)); 3408926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) SLL_SetNext(HardenedSLL::create(ptr), HardenedSLL::null(), central_cache[cl].entropy()); 3409926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) central_cache[cl].InsertRange(HardenedSLL::create(ptr), HardenedSLL::create(ptr), 1); 34105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 34115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } else { 34125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) SpinLockHolder h(&pageheap_lock); 34135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(reinterpret_cast<uintptr_t>(ptr) % kPageSize == 0); 34145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(span != NULL && span->start == p); 34155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#ifndef NO_TCMALLOC_SAMPLES 34165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (span->sample) { 34175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) DLL_Remove(span); 34185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) stacktrace_allocator.Delete(reinterpret_cast<StackTrace*>(span->objects)); 34195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) span->objects = NULL; 34205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 34215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 3422926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 3423926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) POISON_DEALLOCATION(ptr, span->length << kPageShift); 34245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pageheap->Delete(span); 34255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 34265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 34275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 34285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Helpers for use by exported routines below: 34295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 34305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static inline int do_mallopt(int, int) { 34315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 1; // Indicates error 34325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 34335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 34345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#ifdef HAVE_STRUCT_MALLINFO // mallinfo isn't defined on freebsd, for instance 34355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)static inline struct mallinfo do_mallinfo() { 34365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TCMallocStats stats; 34375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ExtractStats(&stats, NULL); 34385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 34395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Just some of the fields are filled in. 34405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) struct mallinfo info; 34415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) memset(&info, 0, sizeof(info)); 34425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 34435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Unfortunately, the struct contains "int" field, so some of the 34445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // size values will be truncated. 34455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) info.arena = static_cast<int>(stats.system_bytes); 34465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) info.fsmblks = static_cast<int>(stats.thread_bytes 34475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) + stats.central_bytes 34485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) + stats.transfer_bytes); 34495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) info.fordblks = static_cast<int>(stats.pageheap_bytes); 34505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) info.uordblks = static_cast<int>(stats.system_bytes 34515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) - stats.thread_bytes 34525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) - stats.central_bytes 34535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) - stats.transfer_bytes 34545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) - stats.pageheap_bytes); 34555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 34565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return info; 34575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 34585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 34595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 34605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)//------------------------------------------------------------------- 34615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Exported routines 34625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)//------------------------------------------------------------------- 34635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 34645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// CAVEAT: The code structure below ensures that MallocHook methods are always 34655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// called from the stack frame of the invoked allocation function. 34665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// heap-checker.cc depends on this to start a stack trace from 34675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// the call to the (de)allocation function. 34685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 34695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void* fastMalloc(size_t size) 34705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 3471f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) return do_malloc(size); 34725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 34735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3474f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles)void fastFree(void* ptr) 34755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 34765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) do_free(ptr); 34775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 34785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 34795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void* fastCalloc(size_t n, size_t elem_size) 34805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 34815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t totalBytes = n * elem_size; 348202772c6a72f1ee0b226341a4f4439970c29fc861Ben Murdoch 34835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Protect against overflow 34845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (n > 1 && elem_size && (totalBytes / elem_size) != n) 34855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 0; 34865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 34875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void* result = do_malloc(totalBytes); 3488f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) memset(result, 0, totalBytes); 34895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 34905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return result; 34915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 34925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 34935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void* fastRealloc(void* old_ptr, size_t new_size) 34945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 34955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (old_ptr == NULL) { 3496f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) return do_malloc(new_size); 34975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 34985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (new_size == 0) { 34995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) free(old_ptr); 35005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return NULL; 35015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 35025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 35035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Get the size of the old entry 35045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const PageID p = reinterpret_cast<uintptr_t>(old_ptr) >> kPageShift; 35055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t cl = pageheap->GetSizeClassIfCached(p); 35065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Span *span = NULL; 35075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t old_size; 35085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (cl == 0) { 35095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) span = pageheap->GetDescriptor(p); 35105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) cl = span->sizeclass; 35115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pageheap->CacheSizeClass(p, cl); 35125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 35135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (cl != 0) { 35145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) old_size = ByteSizeForClass(cl); 35155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } else { 35165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(span != NULL); 35175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) old_size = span->length << kPageShift; 35185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 35195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 35205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Reallocate if the new size is larger than the old size, 35215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // or if the new size is significantly smaller than the old size. 35225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if ((new_size > old_size) || (AllocationSize(new_size) < old_size)) { 35235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Need to reallocate 35245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void* new_ptr = do_malloc(new_size); 35255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) memcpy(new_ptr, old_ptr, ((old_size < new_size) ? old_size : new_size)); 35265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // We could use a variant of do_free() that leverages the fact 35275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // that we already know the sizeclass of old_ptr. The benefit 35285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // would be small, so don't bother. 35295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) do_free(old_ptr); 35305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return new_ptr; 35315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } else { 35325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return old_ptr; 35335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 35345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 35355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 35365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void releaseFastMallocFreeMemory() 35375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 35385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Flush free pages in the current thread cache back to the page heap. 3539926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if (TCMalloc_ThreadCache* threadCache = TCMalloc_ThreadCache::GetCacheIfPresent()) 3540926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) threadCache->Cleanup(); 35415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 35425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) SpinLockHolder h(&pageheap_lock); 35435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pageheap->ReleaseFreePages(); 35445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 3545926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 35465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)FastMallocStatistics fastMallocStatistics() 35475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 35485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) FastMallocStatistics statistics; 35495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 35505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) SpinLockHolder lockHolder(&pageheap_lock); 35515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) statistics.reservedVMBytes = static_cast<size_t>(pageheap->SystemBytes()); 35525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) statistics.committedVMBytes = statistics.reservedVMBytes - pageheap->ReturnedBytes(); 35535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 35545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) statistics.freeListBytes = 0; 35555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (unsigned cl = 0; cl < kNumClasses; ++cl) { 35565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const int length = central_cache[cl].length(); 35575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const int tc_length = central_cache[cl].tc_length(); 35585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 35595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) statistics.freeListBytes += ByteSizeForClass(cl) * (length + tc_length); 35605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 35615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (TCMalloc_ThreadCache* threadCache = thread_heaps; threadCache ; threadCache = threadCache->next_) 35625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) statistics.freeListBytes += threadCache->Size(); 35635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 35645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return statistics; 35655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 35665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 35675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#if OS(DARWIN) 35685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3569926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)template <typename T> 3570926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)T* RemoteMemoryReader::nextEntryInHardenedLinkedList(T** remoteAddress, uintptr_t entropy) const 3571926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles){ 3572926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) T** localAddress = (*this)(remoteAddress); 3573926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if (!localAddress) 3574926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return 0; 3575926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) T* hardenedNext = *localAddress; 3576926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if (!hardenedNext || hardenedNext == (void*)entropy) 3577926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return 0; 3578926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return XOR_MASK_PTR_WITH_KEY(hardenedNext, remoteAddress, entropy); 3579926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)} 3580926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 35815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class FreeObjectFinder { 35825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const RemoteMemoryReader& m_reader; 35835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) HashSet<void*> m_freeObjects; 35845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 35855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)public: 35865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) FreeObjectFinder(const RemoteMemoryReader& reader) : m_reader(reader) { } 35875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 35885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void visit(void* ptr) { m_freeObjects.add(ptr); } 35895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) bool isFreeObject(void* ptr) const { return m_freeObjects.contains(ptr); } 35905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) bool isFreeObject(vm_address_t ptr) const { return isFreeObject(reinterpret_cast<void*>(ptr)); } 35915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t freeObjectCount() const { return m_freeObjects.size(); } 35925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 35935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void findFreeObjects(TCMalloc_ThreadCache* threadCache) 35945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) { 35955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (; threadCache; threadCache = (threadCache->next_ ? m_reader(threadCache->next_) : 0)) 35965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) threadCache->enumerateFreeObjects(*this, m_reader); 35975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 35985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 35995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void findFreeObjects(TCMalloc_Central_FreeListPadded* centralFreeList, size_t numSizes, TCMalloc_Central_FreeListPadded* remoteCentralFreeList) 36005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) { 36015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (unsigned i = 0; i < numSizes; i++) 36025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) centralFreeList[i].enumerateFreeObjects(*this, m_reader, remoteCentralFreeList + i); 36035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 36045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)}; 36055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 36065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class PageMapFreeObjectFinder { 36075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const RemoteMemoryReader& m_reader; 36085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) FreeObjectFinder& m_freeObjectFinder; 3609926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) uintptr_t m_entropy; 36105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 36115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)public: 3612926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) PageMapFreeObjectFinder(const RemoteMemoryReader& reader, FreeObjectFinder& freeObjectFinder, uintptr_t entropy) 36135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) : m_reader(reader) 36145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) , m_freeObjectFinder(freeObjectFinder) 3615926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) , m_entropy(entropy) 3616926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) { 3617926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#if ENABLE(TCMALLOC_HARDENING) 3618926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) ASSERT(m_entropy); 3619926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)#endif 3620926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) } 36215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 36225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int visit(void* ptr) const 36235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) { 36245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (!ptr) 36255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 1; 36265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 36275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Span* span = m_reader(reinterpret_cast<Span*>(ptr)); 36285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (!span) 36295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 1; 36305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 36315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (span->free) { 36325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void* ptr = reinterpret_cast<void*>(span->start << kPageShift); 36335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_freeObjectFinder.visit(ptr); 36345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } else if (span->sizeclass) { 36355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Walk the free list of the small-object span, keeping track of each object seen 3636926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) for (HardenedSLL nextObject = span->objects; nextObject; nextObject.setValue(m_reader.nextEntryInHardenedLinkedList(reinterpret_cast<void**>(nextObject.value()), m_entropy))) 3637926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) m_freeObjectFinder.visit(nextObject.value()); 36385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 36395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return span->length; 36405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 36415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)}; 36425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 36435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class PageMapMemoryUsageRecorder { 36445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) task_t m_task; 36455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void* m_context; 36465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) unsigned m_typeMask; 36475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) vm_range_recorder_t* m_recorder; 36485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const RemoteMemoryReader& m_reader; 36495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const FreeObjectFinder& m_freeObjectFinder; 36505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 36515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) HashSet<void*> m_seenPointers; 36525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Vector<Span*> m_coalescedSpans; 36535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 36545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)public: 36555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) PageMapMemoryUsageRecorder(task_t task, void* context, unsigned typeMask, vm_range_recorder_t* recorder, const RemoteMemoryReader& reader, const FreeObjectFinder& freeObjectFinder) 36565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) : m_task(task) 36575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) , m_context(context) 36585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) , m_typeMask(typeMask) 36595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) , m_recorder(recorder) 36605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) , m_reader(reader) 36615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) , m_freeObjectFinder(freeObjectFinder) 36625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) { } 36635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 36645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ~PageMapMemoryUsageRecorder() 36655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) { 36665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(!m_coalescedSpans.size()); 36675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 36685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 36695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void recordPendingRegions() 36705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) { 3671926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if (!(m_typeMask & (MALLOC_PTR_IN_USE_RANGE_TYPE | MALLOC_PTR_REGION_RANGE_TYPE))) { 36725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_coalescedSpans.clear(); 36735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return; 36745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 36755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 36765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Vector<vm_range_t, 1024> allocatedPointers; 36775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (size_t i = 0; i < m_coalescedSpans.size(); ++i) { 36785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Span *theSpan = m_coalescedSpans[i]; 36795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (theSpan->free) 36805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) continue; 36815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 36825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) vm_address_t spanStartAddress = theSpan->start << kPageShift; 36835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) vm_size_t spanSizeInBytes = theSpan->length * kPageSize; 36845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 36855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (!theSpan->sizeclass) { 36865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // If it's an allocated large object span, mark it as in use 36875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (!m_freeObjectFinder.isFreeObject(spanStartAddress)) 36885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) allocatedPointers.append((vm_range_t){spanStartAddress, spanSizeInBytes}); 36895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } else { 36905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const size_t objectSize = ByteSizeForClass(theSpan->sizeclass); 36915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 36925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Mark each allocated small object within the span as in use 36935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const vm_address_t endOfSpan = spanStartAddress + spanSizeInBytes; 36945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for (vm_address_t object = spanStartAddress; object + objectSize <= endOfSpan; object += objectSize) { 36955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (!m_freeObjectFinder.isFreeObject(object)) 36965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) allocatedPointers.append((vm_range_t){object, objectSize}); 36975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 36985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 36995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 37005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3701926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) (*m_recorder)(m_task, m_context, m_typeMask & (MALLOC_PTR_IN_USE_RANGE_TYPE | MALLOC_PTR_REGION_RANGE_TYPE), allocatedPointers.data(), allocatedPointers.size()); 37025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 37035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_coalescedSpans.clear(); 37045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 37055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 37065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int visit(void* ptr) 37075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) { 37085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (!ptr) 37095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 1; 37105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 37115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Span* span = m_reader(reinterpret_cast<Span*>(ptr)); 37125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (!span || !span->start) 37135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 1; 37145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 37155267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) if (!m_seenPointers.add(ptr).isNewEntry) 37165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return span->length; 37175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 37185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (!m_coalescedSpans.size()) { 37195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_coalescedSpans.append(span); 37205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return span->length; 37215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 37225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 37235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Span* previousSpan = m_coalescedSpans[m_coalescedSpans.size() - 1]; 37245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) vm_address_t previousSpanStartAddress = previousSpan->start << kPageShift; 37255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) vm_size_t previousSpanSizeInBytes = previousSpan->length * kPageSize; 37265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 37275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // If the new span is adjacent to the previous span, do nothing for now. 37285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) vm_address_t spanStartAddress = span->start << kPageShift; 37295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (spanStartAddress == previousSpanStartAddress + previousSpanSizeInBytes) { 37305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_coalescedSpans.append(span); 37315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return span->length; 37325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 37335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 37345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // New span is not adjacent to previous span, so record the spans coalesced so far. 37355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) recordPendingRegions(); 37365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_coalescedSpans.append(span); 37375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 37385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return span->length; 37395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 37405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)}; 37415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 37425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class AdminRegionRecorder { 37435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) task_t m_task; 37445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void* m_context; 37455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) unsigned m_typeMask; 37465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) vm_range_recorder_t* m_recorder; 37475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 37485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Vector<vm_range_t, 1024> m_pendingRegions; 37495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 37505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)public: 37515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) AdminRegionRecorder(task_t task, void* context, unsigned typeMask, vm_range_recorder_t* recorder) 37525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) : m_task(task) 37535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) , m_context(context) 37545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) , m_typeMask(typeMask) 37555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) , m_recorder(recorder) 37565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) { } 37575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 37585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void recordRegion(vm_address_t ptr, size_t size) 37595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) { 37605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (m_typeMask & MALLOC_ADMIN_REGION_RANGE_TYPE) 37615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_pendingRegions.append((vm_range_t){ ptr, size }); 37625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 37635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 37645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void visit(void *ptr, size_t size) 37655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) { 37665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) recordRegion(reinterpret_cast<vm_address_t>(ptr), size); 37675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 37685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 37695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void recordPendingRegions() 37705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) { 37715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if (m_pendingRegions.size()) { 37725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) (*m_recorder)(m_task, m_context, MALLOC_ADMIN_REGION_RANGE_TYPE, m_pendingRegions.data(), m_pendingRegions.size()); 37735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_pendingRegions.clear(); 37745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 37755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 37765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 37775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ~AdminRegionRecorder() 37785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) { 37795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ASSERT(!m_pendingRegions.size()); 37805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 37815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)}; 37825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 37835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)kern_return_t FastMallocZone::enumerate(task_t task, void* context, unsigned typeMask, vm_address_t zoneAddress, memory_reader_t reader, vm_range_recorder_t recorder) 37845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 37855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) RemoteMemoryReader memoryReader(task, reader); 37865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 37875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) InitSizeClasses(); 37885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 37895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) FastMallocZone* mzone = memoryReader(reinterpret_cast<FastMallocZone*>(zoneAddress)); 37905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TCMalloc_PageHeap* pageHeap = memoryReader(mzone->m_pageHeap); 37915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TCMalloc_ThreadCache** threadHeapsPointer = memoryReader(mzone->m_threadHeaps); 37925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TCMalloc_ThreadCache* threadHeaps = memoryReader(*threadHeapsPointer); 37935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 37945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TCMalloc_Central_FreeListPadded* centralCaches = memoryReader(mzone->m_centralCaches, sizeof(TCMalloc_Central_FreeListPadded) * kNumClasses); 37955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 37965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) FreeObjectFinder finder(memoryReader); 37975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) finder.findFreeObjects(threadHeaps); 37985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) finder.findFreeObjects(centralCaches, kNumClasses, mzone->m_centralCaches); 37995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 38005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TCMalloc_PageHeap::PageMap* pageMap = &pageHeap->pagemap_; 3801926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) PageMapFreeObjectFinder pageMapFinder(memoryReader, finder, pageHeap->entropy_); 38025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pageMap->visitValues(pageMapFinder, memoryReader); 38035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 38045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) PageMapMemoryUsageRecorder usageRecorder(task, context, typeMask, recorder, memoryReader, finder); 38055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pageMap->visitValues(usageRecorder, memoryReader); 38065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) usageRecorder.recordPendingRegions(); 38075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 38085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) AdminRegionRecorder adminRegionRecorder(task, context, typeMask, recorder); 38095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pageMap->visitAllocations(adminRegionRecorder, memoryReader); 38105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 38115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) PageHeapAllocator<Span>* spanAllocator = memoryReader(mzone->m_spanAllocator); 38125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) PageHeapAllocator<TCMalloc_ThreadCache>* pageHeapAllocator = memoryReader(mzone->m_pageHeapAllocator); 38135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 38145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) spanAllocator->recordAdministrativeRegions(adminRegionRecorder, memoryReader); 38155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pageHeapAllocator->recordAdministrativeRegions(adminRegionRecorder, memoryReader); 38165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 38175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) adminRegionRecorder.recordPendingRegions(); 38185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 38195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 0; 38205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 38215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 38225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)size_t FastMallocZone::size(malloc_zone_t*, const void*) 38235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 38245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 0; 38255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 38265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 38275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void* FastMallocZone::zoneMalloc(malloc_zone_t*, size_t) 38285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 38295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 0; 38305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 38315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 38325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void* FastMallocZone::zoneCalloc(malloc_zone_t*, size_t, size_t) 38335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 38345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 0; 38355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 38365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 38375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void FastMallocZone::zoneFree(malloc_zone_t*, void* ptr) 38385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 38395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Due to <rdar://problem/5671357> zoneFree may be called by the system free even if the pointer 38405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // is not in this zone. When this happens, the pointer being freed was not allocated by any 38415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // zone so we need to print a useful error for the application developer. 38425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) malloc_printf("*** error for object %p: pointer being freed was not allocated\n", ptr); 38435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 38445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 38455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void* FastMallocZone::zoneRealloc(malloc_zone_t*, void*, size_t) 38465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 38475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 0; 38485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 38495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 38505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 38515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#undef malloc 38525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#undef free 38535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#undef realloc 38545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#undef calloc 38555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 38565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)extern "C" { 38575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)malloc_introspection_t jscore_fastmalloc_introspection = { &FastMallocZone::enumerate, &FastMallocZone::goodSize, &FastMallocZone::check, &FastMallocZone::print, 38585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) &FastMallocZone::log, &FastMallocZone::forceLock, &FastMallocZone::forceUnlock, &FastMallocZone::statistics 38595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 386053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 1060 38615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) , 0 // zone_locked will not be called on the zone unless it advertises itself as version five or higher. 38625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 386353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 1070 38645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) , 0, 0, 0, 0 // These members will not be used unless the zone advertises itself as version seven or higher. 38655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif 38665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 38675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) }; 38685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 38695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 38705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)FastMallocZone::FastMallocZone(TCMalloc_PageHeap* pageHeap, TCMalloc_ThreadCache** threadHeaps, TCMalloc_Central_FreeListPadded* centralCaches, PageHeapAllocator<Span>* spanAllocator, PageHeapAllocator<TCMalloc_ThreadCache>* pageHeapAllocator) 38715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) : m_pageHeap(pageHeap) 38725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) , m_threadHeaps(threadHeaps) 38735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) , m_centralCaches(centralCaches) 38745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) , m_spanAllocator(spanAllocator) 38755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) , m_pageHeapAllocator(pageHeapAllocator) 38765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 38775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) memset(&m_zone, 0, sizeof(m_zone)); 38785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_zone.version = 4; 38795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_zone.zone_name = "JavaScriptCore FastMalloc"; 38805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_zone.size = &FastMallocZone::size; 38815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_zone.malloc = &FastMallocZone::zoneMalloc; 38825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_zone.calloc = &FastMallocZone::zoneCalloc; 38835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_zone.realloc = &FastMallocZone::zoneRealloc; 38845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_zone.free = &FastMallocZone::zoneFree; 38855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_zone.valloc = &FastMallocZone::zoneValloc; 38865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_zone.destroy = &FastMallocZone::zoneDestroy; 38875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) m_zone.introspect = &jscore_fastmalloc_introspection; 38885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) malloc_zone_register(&m_zone); 38895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 38905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 38915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 38925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)void FastMallocZone::init() 38935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 38945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) static FastMallocZone zone(pageheap, &thread_heaps, static_cast<TCMalloc_Central_FreeListPadded*>(central_cache), &span_allocator, &threadheap_allocator); 38955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} 38965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 38975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif // OS(DARWIN) 38985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 38995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)} // namespace WTF 39005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 39015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#endif // FORCE_SYSTEM_MALLOC 3902