15821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Copyright (c) 2008, Google Inc. 25821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// All rights reserved. 35821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 45821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Redistribution and use in source and binary forms, with or without 55821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// modification, are permitted provided that the following conditions are 65821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// met: 75821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 85821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// * Redistributions of source code must retain the above copyright 95821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// notice, this list of conditions and the following disclaimer. 105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// * Redistributions in binary form must reproduce the above 115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// copyright notice, this list of conditions and the following disclaimer 125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// in the documentation and/or other materials provided with the 135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// distribution. 145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// * Neither the name of Google Inc. nor the names of its 155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// contributors may be used to endorse or promote products derived from 165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// this software without specific prior written permission. 175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// --- 315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Author: Sanjay Ghemawat <opensource@google.com> 325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifndef TCMALLOC_THREAD_CACHE_H_ 345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define TCMALLOC_THREAD_CACHE_H_ 355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <config.h> 375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifdef HAVE_PTHREAD 385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <pthread.h> // for pthread_t, pthread_key_t 395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <stddef.h> // for size_t, NULL 415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifdef HAVE_STDINT_H 425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <stdint.h> // for uint32_t, uint64_t 435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <sys/types.h> // for ssize_t 455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "common.h" // for SizeMap, kMaxSize, etc 465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "free_list.h" // for FL_Pop, FL_PopRange, etc 475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "internal_logging.h" // for ASSERT, etc 485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "maybe_threads.h" 495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "page_heap_allocator.h" // for PageHeapAllocator 505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "sampler.h" // for Sampler 515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "static_vars.h" // for Static 525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)namespace tcmalloc { 545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Even if we have support for thread-local storage in the compiler 565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// and linker, the OS may not support it. We need to check that at 575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// runtime. Right now, we have to keep a manual set of "bad" OSes. 585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if defined(HAVE_TLS) 595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern bool kernel_supports_tls; // defined in thread_cache.cc 605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void CheckIfKernelSupportsTLS(); 615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline bool KernelSupportsTLS() { 625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return kernel_supports_tls; 635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif // HAVE_TLS 655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)//------------------------------------------------------------------- 675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Data kept per thread 685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)//------------------------------------------------------------------- 695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class ThreadCache { 715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) public: 725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // All ThreadCache objects are kept in a linked list (for stats collection) 735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ThreadCache* next_; 745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ThreadCache* prev_; 755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void Init(pthread_t tid); 775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void Cleanup(); 785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Accessors (mostly just for printing stats) 805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int freelist_length(size_t cl) const { return list_[cl].length(); } 815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Total byte size in cache 835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t Size() const { return size_; } 845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Allocate an object of the given size and class. The size given 865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // must be the same as the size of the class in the size map. 875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* Allocate(size_t size, size_t cl); 885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void Deallocate(void* ptr, size_t size_class); 895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void Scavenge(); 915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int GetSamplePeriod(); 935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Record allocation of "k" bytes. Return true iff allocation 955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // should be sampled 965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) bool SampleAllocation(size_t k); 975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Record additional bytes allocated. 995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void AddToByteAllocatedTotal(size_t k) { total_bytes_allocated_ += k; } 1005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Return the total number of bytes allocated from this heap. The value will 1025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // wrap when there is an overflow, and so only the differences between two 1035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // values should be relied on (and even then, modulo 2^32). 1045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint32 GetTotalBytesAllocated() const; 1055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // On the current thread, return GetTotalBytesAllocated(). 1075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static uint32 GetBytesAllocatedOnCurrentThread(); 1085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static void InitModule(); 1105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static void InitTSD(); 1115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static ThreadCache* GetThreadHeap(); 1125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static ThreadCache* GetCache(); 1135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static ThreadCache* GetCacheIfPresent(); 1145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static ThreadCache* CreateCacheIfNecessary(); 1155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static void BecomeIdle(); 1165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Return the number of thread heaps in use. 1185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static inline int HeapsInUse(); 1195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Writes to total_bytes the total number of bytes used by all thread heaps. 1215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // class_count must be an array of size kNumClasses. Writes the number of 1225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // items on the corresponding freelist. class_count may be NULL. 1235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // The storage of both parameters must be zero intialized. 1245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // REQUIRES: Static::pageheap_lock is held. 1255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static void GetThreadStats(uint64_t* total_bytes, uint64_t* class_count); 1265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Sets the total thread cache size to new_size, recomputing the 1285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // individual thread cache sizes as necessary. 1295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // REQUIRES: Static::pageheap lock is held. 1305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static void set_overall_thread_cache_size(size_t new_size); 1315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static size_t overall_thread_cache_size() { 1325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return overall_thread_cache_size_; 1335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) private: 1365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) class FreeList { 1375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) private: 1385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* list_; // Linked list of nodes 1395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifdef _LP64 1415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // On 64-bit hardware, manipulating 16-bit values may be slightly slow. 1425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint32_t length_; // Current length. 1435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint32_t lowater_; // Low water mark for list length. 1445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint32_t max_length_; // Dynamic max list length based on usage. 1455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Tracks the number of times a deallocation has caused 1465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // length_ > max_length_. After the kMaxOverages'th time, max_length_ 1475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // shrinks and length_overages_ is reset to zero. 1485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint32_t length_overages_; 1495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#else 1505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // If we aren't using 64-bit pointers then pack these into less space. 1515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint16_t length_; 1525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint16_t lowater_; 1535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint16_t max_length_; 1545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint16_t length_overages_; 1555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 1565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) public: 1585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void Init() { 1595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) list_ = NULL; 1605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) length_ = 0; 1615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) lowater_ = 0; 1625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) max_length_ = 1; 1635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) length_overages_ = 0; 1645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Return current length of list 1675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t length() const { 1685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return length_; 1695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Return the maximum length of the list. 1725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t max_length() const { 1735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return max_length_; 1745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Set the maximum length of the list. If 'new_max' > length(), the 1775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // client is responsible for removing objects from the list. 1785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void set_max_length(size_t new_max) { 1795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) max_length_ = new_max; 1805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Return the number of times that length() has gone over max_length(). 1835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t length_overages() const { 1845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return length_overages_; 1855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void set_length_overages(size_t new_count) { 1885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) length_overages_ = new_count; 1895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Is list empty? 1925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) bool empty() const { 1935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return list_ == NULL; 1945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Low-water mark management 1975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int lowwatermark() const { return lowater_; } 1985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void clear_lowwatermark() { lowater_ = length_; } 1995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void Push(void* ptr) { 2015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FL_Push(&list_, ptr); 2025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) length_++; 2035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* Pop() { 2065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ASSERT(list_ != NULL); 2075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) length_--; 2085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (length_ < lowater_) lowater_ = length_; 2095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return FL_Pop(&list_); 2105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* Next() { 2135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (list_ == NULL) return NULL; 2145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return FL_Next(list_); 2155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void PushRange(int N, void *start, void *end) { 2185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FL_PushRange(&list_, start, end); 2195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) length_ += N; 2205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void PopRange(int N, void **start, void **end) { 2235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FL_PopRange(&list_, N, start, end); 2245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ASSERT(length_ >= N); 2255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) length_ -= N; 2265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (length_ < lowater_) lowater_ = length_; 2275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }; 2295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Gets and returns an object from the central cache, and, if possible, 2315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // also adds some objects of that size class to this thread cache. 2325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void* FetchFromCentralCache(size_t cl, size_t byte_size); 2335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Releases some number of items from src. Adjusts the list's max_length 2355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // to eventually converge on num_objects_to_move(cl). 2365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void ListTooLong(FreeList* src, size_t cl); 2375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Releases N items from this thread cache. 2395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void ReleaseToCentralCache(FreeList* src, size_t cl, int N); 2405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Increase max_size_ by reducing unclaimed_cache_space_ or by 2425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // reducing the max_size_ of some other thread. In both cases, 2435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // the delta is kStealAmount. 2445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void IncreaseCacheLimit(); 2455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Same as above but requires Static::pageheap_lock() is held. 2465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void IncreaseCacheLimitLocked(); 2475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // If TLS is available, we also store a copy of the per-thread object 2495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // in a __thread variable since __thread variables are faster to read 2505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // than pthread_getspecific(). We still need pthread_setspecific() 2515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // because __thread variables provide no way to run cleanup code when 2525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // a thread is destroyed. 2535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // We also give a hint to the compiler to use the "initial exec" TLS 2545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // model. This is faster than the default TLS model, at the cost that 2555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // you cannot dlopen this library. (To see the difference, look at 2565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // the CPU use of __tls_get_addr with and without this attribute.) 2575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Since we don't really use dlopen in google code -- and using dlopen 2585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // on a malloc replacement is asking for trouble in any case -- that's 2595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // a good tradeoff for us. 2605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifdef HAVE_TLS 2615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static __thread ThreadCache* threadlocal_heap_ 2625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // This code links against pyautolib.so, which causes dlopen() on that shared 2635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // object to fail when -fprofile-generate is used with it. Ideally 2645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // pyautolib.so should not link against this code. There is a bug filed for 2655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // that: 2665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // http://code.google.com/p/chromium/issues/detail?id=124489 2675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // For now the workaround is to pass in -DPGO_GENERATE when building Chrome 2685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // for instrumentation (-fprofile-generate). 2695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // For all non-instrumentation builds, this define will not be set and the 2705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // performance benefit of "intial-exec" will be achieved. 2715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if defined(HAVE___ATTRIBUTE__) && !defined(PGO_GENERATE) 2725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) __attribute__ ((tls_model ("initial-exec"))) 2735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# endif 2745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ; 2755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 2765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Thread-specific key. Initialization here is somewhat tricky 2785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // because some Linux startup code invokes malloc() before it 2795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // is in a good enough state to handle pthread_keycreate(). 2805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Therefore, we use TSD keys only after tsd_inited is set to true. 2815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Until then, we use a slow path to get the heap object. 2825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static bool tsd_inited_; 2835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static pthread_key_t heap_key_; 2845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Linked list of heap objects. Protected by Static::pageheap_lock. 2865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static ThreadCache* thread_heaps_; 2875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static int thread_heap_count_; 2885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // A pointer to one of the objects in thread_heaps_. Represents 2905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // the next ThreadCache from which a thread over its max_size_ should 2915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // steal memory limit. Round-robin through all of the objects in 2925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // thread_heaps_. Protected by Static::pageheap_lock. 2935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static ThreadCache* next_memory_steal_; 2945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Overall thread cache size. Protected by Static::pageheap_lock. 2965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static size_t overall_thread_cache_size_; 2975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Global per-thread cache size. Writes are protected by 2995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Static::pageheap_lock. Reads are done without any locking, which should be 3005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // fine as long as size_t can be written atomically and we don't place 3015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // invariants between this variable and other pieces of state. 3025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static volatile size_t per_thread_cache_size_; 3035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Represents overall_thread_cache_size_ minus the sum of max_size_ 3055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // across all ThreadCaches. Protected by Static::pageheap_lock. 3065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static ssize_t unclaimed_cache_space_; 3075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // This class is laid out with the most frequently used fields 3095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // first so that hot elements are placed on the same cache line. 3105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t size_; // Combined size of data 3125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_t max_size_; // size_ > max_size_ --> Scavenge() 3135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // The following is the tally of bytes allocated on a thread as a response to 3155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // any flavor of malloc() call. The aggegated amount includes all padding to 3165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // the smallest class that can hold the request, or to the nearest whole page 3175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // when a large allocation is made without using a class. This sum is 3185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // currently used for Chromium profiling, where tallies are kept of the amount 3195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // of memory allocated during the running of each task on each thread. 3205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uint32 total_bytes_allocated_; // Total, modulo 2^32. 3215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // We sample allocations, biased by the size of the allocation 3235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Sampler sampler_; // A sampler 3245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FreeList list_[kNumClasses]; // Array indexed by size-class 3265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pthread_t tid_; // Which thread owns it 3285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) bool in_setspecific_; // In call to pthread_setspecific? 3295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Allocate a new heap. REQUIRES: Static::pageheap_lock is held. 3315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static ThreadCache* NewHeap(pthread_t tid); 3325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Use only as pthread thread-specific destructor function. 3345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static void DestroyThreadCache(void* ptr); 3355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static void DeleteCache(ThreadCache* heap); 3375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static void RecomputePerThreadCacheSize(); 3385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Ensure that this class is cacheline-aligned. This is critical for 3405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // performance, as false sharing would negate many of the benefits 3415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // of a per-thread cache. 3425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} CACHELINE_ALIGNED; 3435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Allocator for thread heaps 3455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// This is logically part of the ThreadCache class, but MSVC, at 3465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// least, does not like using ThreadCache as a template argument 3475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// before the class is fully defined. So we put it outside the class. 3485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern PageHeapAllocator<ThreadCache> threadcache_allocator; 3495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline int ThreadCache::HeapsInUse() { 3515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return threadcache_allocator.inuse(); 3525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline bool ThreadCache::SampleAllocation(size_t k) { 3555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return sampler_.SampleAllocation(k); 3565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline uint32 ThreadCache::GetTotalBytesAllocated() const { 3595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return total_bytes_allocated_; 3605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void* ThreadCache::Allocate(size_t size, size_t cl) { 3635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ASSERT(size <= kMaxSize); 3645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ASSERT(size == Static::sizemap()->ByteSizeForClass(cl)); 3655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FreeList* list = &list_[cl]; 3675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (list->empty()) { 3685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return FetchFromCentralCache(cl, size); 3695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 3705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_ -= size; 3715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return list->Pop(); 3725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void ThreadCache::Deallocate(void* ptr, size_t cl) { 3755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FreeList* list = &list_[cl]; 3765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) size_ += Static::sizemap()->ByteSizeForClass(cl); 3775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ssize_t size_headroom = max_size_ - size_ - 1; 3785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // This catches back-to-back frees of allocs in the same size 3805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // class. A more comprehensive (and expensive) test would be to walk 3815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // the entire freelist. But this might be enough to find some bugs. 3825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ASSERT(ptr != list->Next()); 3835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) list->Push(ptr); 3855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ssize_t list_headroom = 3865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static_cast<ssize_t>(list->max_length()) - list->length(); 3875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // There are two relatively uncommon things that require further work. 3895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // In the common case we're done, and in that case we need a single branch 3905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // because of the bitwise-or trick that follows. 3915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if ((list_headroom | size_headroom) < 0) { 3925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (list_headroom < 0) { 3935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ListTooLong(list, cl); 3945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 3955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (size_ >= max_size_) Scavenge(); 3965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 3975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline ThreadCache* ThreadCache::GetThreadHeap() { 4005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifdef HAVE_TLS 4015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // __thread is faster, but only when the kernel supports it 4025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (KernelSupportsTLS()) 4035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return threadlocal_heap_; 4045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 4055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return reinterpret_cast<ThreadCache *>( 4065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) perftools_pthread_getspecific(heap_key_)); 4075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 4085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline ThreadCache* ThreadCache::GetCache() { 4105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ThreadCache* ptr = NULL; 4115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (!tsd_inited_) { 4125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) InitModule(); 4135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } else { 4145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ptr = GetThreadHeap(); 4155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (ptr == NULL) ptr = CreateCacheIfNecessary(); 4175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ptr; 4185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 4195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// In deletion paths, we do not try to create a thread-cache. This is 4215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// because we may be in the thread destruction code and may have 4225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// already cleaned up the cache for this thread. 4235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline ThreadCache* ThreadCache::GetCacheIfPresent() { 4245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (!tsd_inited_) return NULL; 4255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return GetThreadHeap(); 4265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 4275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} // namespace tcmalloc 4295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif // TCMALLOC_THREAD_CACHE_H_ 431