1// Copyright (c) 2012 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include "base/allocator/allocator_shim.h" 6 7#include <config.h> 8#include "base/allocator/allocator_extension_thunks.h" 9#include "base/profiler/alternate_timer.h" 10#include "base/sysinfo.h" 11#include "jemalloc.h" 12 13// When defined, different heap allocators can be used via an environment 14// variable set before running the program. This may reduce the amount 15// of inlining that we get with malloc/free/etc. Disabling makes it 16// so that only tcmalloc can be used. 17#define ENABLE_DYNAMIC_ALLOCATOR_SWITCHING 18 19// TODO(mbelshe): Ensure that all calls to tcmalloc have the proper call depth 20// from the "user code" so that debugging tools (HeapChecker) can work. 21 22// __THROW is defined in glibc systems. It means, counter-intuitively, 23// "This function will never throw an exception." It's an optional 24// optimization tool, but we may need to use it to match glibc prototypes. 25#ifndef __THROW // I guess we're not on a glibc system 26# define __THROW // __THROW is just an optimization, so ok to make it "" 27#endif 28 29// new_mode behaves similarly to MSVC's _set_new_mode. 30// If flag is 0 (default), calls to malloc will behave normally. 31// If flag is 1, calls to malloc will behave like calls to new, 32// and the std_new_handler will be invoked on failure. 33// Can be set by calling _set_new_mode(). 34static int new_mode = 0; 35 36typedef enum { 37 TCMALLOC, // TCMalloc is the default allocator. 38 JEMALLOC, // JEMalloc. 39 WINHEAP, // Windows Heap (standard Windows allocator). 40 WINLFH, // Windows LFH Heap. 41} Allocator; 42 43// This is the default allocator. This value can be changed at startup by 44// specifying environment variables shown below it. 45// See SetupSubprocessAllocator() to specify a default secondary (subprocess) 46// allocator. 47// TODO(jar): Switch to using TCMALLOC for the renderer as well. 48#if (defined(ADDRESS_SANITIZER) && defined(OS_WIN)) 49// The Windows implementation of Asan requires the use of "WINHEAP". 50static Allocator allocator = WINHEAP; 51#else 52static Allocator allocator = TCMALLOC; 53#endif 54// The names of the environment variables that can optionally control the 55// selection of the allocator. The primary may be used to control overall 56// allocator selection, and the secondary can be used to specify an allocator 57// to use in sub-processes. 58static const char primary_name[] = "CHROME_ALLOCATOR"; 59static const char secondary_name[] = "CHROME_ALLOCATOR_2"; 60 61// We include tcmalloc and the win_allocator to get as much inlining as 62// possible. 63#include "debugallocation_shim.cc" 64#include "win_allocator.cc" 65 66// Forward declarations from jemalloc. 67extern "C" { 68void* je_malloc(size_t s); 69void* je_realloc(void* p, size_t s); 70void je_free(void* s); 71size_t je_msize(void* p); 72bool je_malloc_init_hard(); 73void* je_memalign(size_t a, size_t s); 74} 75 76// Call the new handler, if one has been set. 77// Returns true on successfully calling the handler, false otherwise. 78inline bool call_new_handler(bool nothrow) { 79 // Get the current new handler. NB: this function is not 80 // thread-safe. We make a feeble stab at making it so here, but 81 // this lock only protects against tcmalloc interfering with 82 // itself, not with other libraries calling set_new_handler. 83 std::new_handler nh; 84 { 85 SpinLockHolder h(&set_new_handler_lock); 86 nh = std::set_new_handler(0); 87 (void) std::set_new_handler(nh); 88 } 89#if (defined(__GNUC__) && !defined(__EXCEPTIONS)) || \ 90 (defined(_HAS_EXCEPTIONS) && !_HAS_EXCEPTIONS) 91 if (!nh) 92 return false; 93 // Since exceptions are disabled, we don't really know if new_handler 94 // failed. Assume it will abort if it fails. 95 (*nh)(); 96 return false; // break out of the retry loop. 97#else 98 // If no new_handler is established, the allocation failed. 99 if (!nh) { 100 if (nothrow) 101 return false; 102 throw std::bad_alloc(); 103 } 104 // Otherwise, try the new_handler. If it returns, retry the 105 // allocation. If it throws std::bad_alloc, fail the allocation. 106 // if it throws something else, don't interfere. 107 try { 108 (*nh)(); 109 } catch (const std::bad_alloc&) { 110 if (!nothrow) 111 throw; 112 return true; 113 } 114#endif // (defined(__GNUC__) && !defined(__EXCEPTIONS)) || (defined(_HAS_EXCEPTIONS) && !_HAS_EXCEPTIONS) 115 return false; 116} 117 118extern "C" { 119void* malloc(size_t size) __THROW { 120 void* ptr; 121 for (;;) { 122#ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING 123 switch (allocator) { 124 case JEMALLOC: 125 ptr = je_malloc(size); 126 break; 127 case WINHEAP: 128 case WINLFH: 129 ptr = win_heap_malloc(size); 130 break; 131 case TCMALLOC: 132 default: 133 ptr = do_malloc(size); 134 break; 135 } 136#else 137 // TCMalloc case. 138 ptr = do_malloc(size); 139#endif 140 if (ptr) 141 return ptr; 142 143 if (!new_mode || !call_new_handler(true)) 144 break; 145 } 146 return ptr; 147} 148 149void free(void* p) __THROW { 150#ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING 151 switch (allocator) { 152 case JEMALLOC: 153 je_free(p); 154 return; 155 case WINHEAP: 156 case WINLFH: 157 win_heap_free(p); 158 return; 159 } 160#endif 161 // TCMalloc case. 162 do_free(p); 163} 164 165void* realloc(void* ptr, size_t size) __THROW { 166 // Webkit is brittle for allocators that return NULL for malloc(0). The 167 // realloc(0, 0) code path does not guarantee a non-NULL return, so be sure 168 // to call malloc for this case. 169 if (!ptr) 170 return malloc(size); 171 172 void* new_ptr; 173 for (;;) { 174#ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING 175 switch (allocator) { 176 case JEMALLOC: 177 new_ptr = je_realloc(ptr, size); 178 break; 179 case WINHEAP: 180 case WINLFH: 181 new_ptr = win_heap_realloc(ptr, size); 182 break; 183 case TCMALLOC: 184 default: 185 new_ptr = do_realloc(ptr, size); 186 break; 187 } 188#else 189 // TCMalloc case. 190 new_ptr = do_realloc(ptr, size); 191#endif 192 193 // Subtle warning: NULL return does not alwas indicate out-of-memory. If 194 // the requested new size is zero, realloc should free the ptr and return 195 // NULL. 196 if (new_ptr || !size) 197 return new_ptr; 198 if (!new_mode || !call_new_handler(true)) 199 break; 200 } 201 return new_ptr; 202} 203 204// TODO(mbelshe): Implement this for other allocators. 205void malloc_stats(void) __THROW { 206#ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING 207 switch (allocator) { 208 case JEMALLOC: 209 // No stats. 210 return; 211 case WINHEAP: 212 case WINLFH: 213 // No stats. 214 return; 215 } 216#endif 217 tc_malloc_stats(); 218} 219 220#ifdef WIN32 221 222extern "C" size_t _msize(void* p) { 223#ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING 224 switch (allocator) { 225 case JEMALLOC: 226 return je_msize(p); 227 case WINHEAP: 228 case WINLFH: 229 return win_heap_msize(p); 230 } 231#endif 232 return MallocExtension::instance()->GetAllocatedSize(p); 233} 234 235// This is included to resolve references from libcmt. 236extern "C" intptr_t _get_heap_handle() { 237 return 0; 238} 239 240static bool get_allocator_waste_size_thunk(size_t* size) { 241#ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING 242 switch (allocator) { 243 case JEMALLOC: 244 case WINHEAP: 245 case WINLFH: 246 // TODO(alexeif): Implement for allocators other than tcmalloc. 247 return false; 248 } 249#endif 250 size_t heap_size, allocated_bytes, unmapped_bytes; 251 MallocExtension* ext = MallocExtension::instance(); 252 if (ext->GetNumericProperty("generic.heap_size", &heap_size) && 253 ext->GetNumericProperty("generic.current_allocated_bytes", 254 &allocated_bytes) && 255 ext->GetNumericProperty("tcmalloc.pageheap_unmapped_bytes", 256 &unmapped_bytes)) { 257 *size = heap_size - allocated_bytes - unmapped_bytes; 258 return true; 259 } 260 return false; 261} 262 263static void get_stats_thunk(char* buffer, int buffer_length) { 264 MallocExtension::instance()->GetStats(buffer, buffer_length); 265} 266 267static void release_free_memory_thunk() { 268 MallocExtension::instance()->ReleaseFreeMemory(); 269} 270 271// The CRT heap initialization stub. 272extern "C" int _heap_init() { 273#ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING 274// Don't use the environment variable if ADDRESS_SANITIZER is defined on 275// Windows, as the implementation requires Winheap to be the allocator. 276#if !(defined(ADDRESS_SANITIZER) && defined(OS_WIN)) 277 const char* environment_value = GetenvBeforeMain(primary_name); 278 if (environment_value) { 279 if (!stricmp(environment_value, "jemalloc")) 280 allocator = JEMALLOC; 281 else if (!stricmp(environment_value, "winheap")) 282 allocator = WINHEAP; 283 else if (!stricmp(environment_value, "winlfh")) 284 allocator = WINLFH; 285 else if (!stricmp(environment_value, "tcmalloc")) 286 allocator = TCMALLOC; 287 } 288#endif 289 290 switch (allocator) { 291 case JEMALLOC: 292 return je_malloc_init_hard() ? 0 : 1; 293 case WINHEAP: 294 return win_heap_init(false) ? 1 : 0; 295 case WINLFH: 296 return win_heap_init(true) ? 1 : 0; 297 case TCMALLOC: 298 default: 299 // fall through 300 break; 301 } 302#endif 303 // Initializing tcmalloc. 304 // We intentionally leak this object. It lasts for the process 305 // lifetime. Trying to teardown at _heap_term() is so late that 306 // you can't do anything useful anyway. 307 new TCMallocGuard(); 308 309 // Provide optional hook for monitoring allocation quantities on a per-thread 310 // basis. Only set the hook if the environment indicates this needs to be 311 // enabled. 312 const char* profiling = 313 GetenvBeforeMain(tracked_objects::kAlternateProfilerTime); 314 if (profiling && *profiling == '1') { 315 tracked_objects::SetAlternateTimeSource( 316 tcmalloc::ThreadCache::GetBytesAllocatedOnCurrentThread, 317 tracked_objects::TIME_SOURCE_TYPE_TCMALLOC); 318 } 319 320 base::allocator::thunks::SetGetAllocatorWasteSizeFunction( 321 get_allocator_waste_size_thunk); 322 base::allocator::thunks::SetGetStatsFunction(get_stats_thunk); 323 base::allocator::thunks::SetReleaseFreeMemoryFunction( 324 release_free_memory_thunk); 325 326 return 1; 327} 328 329// The CRT heap cleanup stub. 330extern "C" void _heap_term() {} 331 332// We set this to 1 because part of the CRT uses a check of _crtheap != 0 333// to test whether the CRT has been initialized. Once we've ripped out 334// the allocators from libcmt, we need to provide this definition so that 335// the rest of the CRT is still usable. 336extern "C" void* _crtheap = reinterpret_cast<void*>(1); 337 338// Provide support for aligned memory through Windows only _aligned_malloc(). 339void* _aligned_malloc(size_t size, size_t alignment) { 340 // _aligned_malloc guarantees parameter validation, so do so here. These 341 // checks are somewhat stricter than _aligned_malloc() since we're effectively 342 // using memalign() under the hood. 343 DCHECK_GT(size, 0U); 344 DCHECK_EQ(alignment & (alignment - 1), 0U); 345 DCHECK_EQ(alignment % sizeof(void*), 0U); 346 347 void* ptr; 348 for (;;) { 349#ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING 350 switch (allocator) { 351 case JEMALLOC: 352 ptr = je_memalign(alignment, size); 353 break; 354 case WINHEAP: 355 case WINLFH: 356 ptr = win_heap_memalign(alignment, size); 357 break; 358 case TCMALLOC: 359 default: 360 ptr = tc_memalign(alignment, size); 361 break; 362 } 363#else 364 // TCMalloc case. 365 ptr = tc_memalign(alignment, size); 366#endif 367 if (ptr) { 368 // Sanity check alignment. 369 DCHECK_EQ(reinterpret_cast<uintptr_t>(ptr) & (alignment - 1), 0U); 370 return ptr; 371 } 372 373 if (!new_mode || !call_new_handler(true)) 374 break; 375 } 376 return ptr; 377} 378 379void _aligned_free(void* p) { 380 // Both JEMalloc and TCMalloc return pointers from memalign() that are safe to 381 // use with free(). Pointers allocated with win_heap_memalign() MUST be freed 382 // via win_heap_memalign_free() since the aligned pointer is not the real one. 383#ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING 384 switch (allocator) { 385 case JEMALLOC: 386 je_free(p); 387 return; 388 case WINHEAP: 389 case WINLFH: 390 win_heap_memalign_free(p); 391 return; 392 } 393#endif 394 // TCMalloc case. 395 do_free(p); 396} 397 398#endif // WIN32 399 400#include "generic_allocators.cc" 401 402} // extern C 403 404namespace base { 405namespace allocator { 406 407void SetupSubprocessAllocator() { 408#ifdef ENABLE_DYNAMIC_ALLOCATOR_SWITCHING 409 size_t primary_length = 0; 410 getenv_s(&primary_length, NULL, 0, primary_name); 411 412 size_t secondary_length = 0; 413 char buffer[20]; 414 getenv_s(&secondary_length, buffer, sizeof(buffer), secondary_name); 415 DCHECK_GT(sizeof(buffer), secondary_length); 416 buffer[sizeof(buffer) - 1] = '\0'; 417 418 if (secondary_length || !primary_length) { 419 // Don't use the environment variable if ADDRESS_SANITIZER is defined on 420 // Windows, as the implementation require Winheap to be the allocator. 421#if !(defined(ADDRESS_SANITIZER) && defined(OS_WIN)) 422 const char* secondary_value = secondary_length ? buffer : "TCMALLOC"; 423 // Force renderer (or other subprocesses) to use secondary_value. 424#else 425 const char* secondary_value = "WINHEAP"; 426#endif 427 int ret_val = _putenv_s(primary_name, secondary_value); 428 DCHECK_EQ(0, ret_val); 429 } 430#endif // ENABLE_DYNAMIC_ALLOCATOR_SWITCHING 431} 432 433void* TCMallocDoMallocForTest(size_t size) { 434 return do_malloc(size); 435} 436 437void TCMallocDoFreeForTest(void* ptr) { 438 do_free(ptr); 439} 440 441size_t ExcludeSpaceForMarkForTest(size_t size) { 442 return ExcludeSpaceForMark(size); 443} 444 445} // namespace allocator. 446} // namespace base. 447