1// Copyright (c) 2005, Google Inc. 2// All rights reserved. 3// 4// Redistribution and use in source and binary forms, with or without 5// modification, are permitted provided that the following conditions are 6// met: 7// 8// * Redistributions of source code must retain the above copyright 9// notice, this list of conditions and the following disclaimer. 10// * Redistributions in binary form must reproduce the above 11// copyright notice, this list of conditions and the following disclaimer 12// in the documentation and/or other materials provided with the 13// distribution. 14// * Neither the name of Google Inc. nor the names of its 15// contributors may be used to endorse or promote products derived from 16// this software without specific prior written permission. 17// 18// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30// --- 31// Author: Sanjay Ghemawat <opensource@google.com> 32 33#include <config.h> 34 35// Disable the glibc prototype of mremap(), as older versions of the 36// system headers define this function with only four arguments, 37// whereas newer versions allow an optional fifth argument: 38#ifdef HAVE_MMAP 39# define mremap glibc_mremap 40# include <sys/mman.h> 41# undef mremap 42#endif 43 44#include <stddef.h> 45#ifdef HAVE_STDINT_H 46#include <stdint.h> 47#endif 48#include <algorithm> 49#include "base/logging.h" 50#include "base/spinlock.h" 51#include "maybe_threads.h" 52#include "malloc_hook-inl.h" 53#include <gperftools/malloc_hook.h> 54 55// This #ifdef should almost never be set. Set NO_TCMALLOC_SAMPLES if 56// you're porting to a system where you really can't get a stacktrace. 57#ifdef NO_TCMALLOC_SAMPLES 58 // We use #define so code compiles even if you #include stacktrace.h somehow. 59# define GetStackTrace(stack, depth, skip) (0) 60#else 61# include <gperftools/stacktrace.h> 62#endif 63 64// __THROW is defined in glibc systems. It means, counter-intuitively, 65// "This function will never throw an exception." It's an optional 66// optimization tool, but we may need to use it to match glibc prototypes. 67#ifndef __THROW // I guess we're not on a glibc system 68# define __THROW // __THROW is just an optimization, so ok to make it "" 69#endif 70 71using std::copy; 72 73 74// Declaration of default weak initialization function, that can be overridden 75// by linking-in a strong definition (as heap-checker.cc does). This is 76// extern "C" so that it doesn't trigger gold's --detect-odr-violations warning, 77// which only looks at C++ symbols. 78// 79// This function is declared here as weak, and defined later, rather than a more 80// straightforward simple weak definition, as a workround for an icc compiler 81// issue ((Intel reference 290819). This issue causes icc to resolve weak 82// symbols too early, at compile rather than link time. By declaring it (weak) 83// here, then defining it below after its use, we can avoid the problem. 84extern "C" { 85ATTRIBUTE_WEAK void MallocHook_InitAtFirstAllocation_HeapLeakChecker(); 86} 87 88namespace { 89 90void RemoveInitialHooksAndCallInitializers(); // below. 91 92pthread_once_t once = PTHREAD_ONCE_INIT; 93 94// These hooks are installed in MallocHook as the only initial hooks. The first 95// hook that is called will run RemoveInitialHooksAndCallInitializers (see the 96// definition below) and then redispatch to any malloc hooks installed by 97// RemoveInitialHooksAndCallInitializers. 98// 99// Note(llib): there is a possibility of a race in the event that there are 100// multiple threads running before the first allocation. This is pretty 101// difficult to achieve, but if it is then multiple threads may concurrently do 102// allocations. The first caller will call 103// RemoveInitialHooksAndCallInitializers via one of the initial hooks. A 104// concurrent allocation may, depending on timing either: 105// * still have its initial malloc hook installed, run that and block on waiting 106// for the first caller to finish its call to 107// RemoveInitialHooksAndCallInitializers, and proceed normally. 108// * occur some time during the RemoveInitialHooksAndCallInitializers call, at 109// which point there could be no initial hooks and the subsequent hooks that 110// are about to be set up by RemoveInitialHooksAndCallInitializers haven't 111// been installed yet. I think the worst we can get is that some allocations 112// will not get reported to some hooks set by the initializers called from 113// RemoveInitialHooksAndCallInitializers. 114 115void InitialNewHook(const void* ptr, size_t size) { 116 perftools_pthread_once(&once, &RemoveInitialHooksAndCallInitializers); 117 MallocHook::InvokeNewHook(ptr, size); 118} 119 120void InitialPreMMapHook(const void* start, 121 size_t size, 122 int protection, 123 int flags, 124 int fd, 125 off_t offset) { 126 perftools_pthread_once(&once, &RemoveInitialHooksAndCallInitializers); 127 MallocHook::InvokePreMmapHook(start, size, protection, flags, fd, offset); 128} 129 130void InitialPreSbrkHook(ptrdiff_t increment) { 131 perftools_pthread_once(&once, &RemoveInitialHooksAndCallInitializers); 132 MallocHook::InvokePreSbrkHook(increment); 133} 134 135// This function is called at most once by one of the above initial malloc 136// hooks. It removes all initial hooks and initializes all other clients that 137// want to get control at the very first memory allocation. The initializers 138// may assume that the initial malloc hooks have been removed. The initializers 139// may set up malloc hooks and allocate memory. 140void RemoveInitialHooksAndCallInitializers() { 141 RAW_CHECK(MallocHook::RemoveNewHook(&InitialNewHook), ""); 142 RAW_CHECK(MallocHook::RemovePreMmapHook(&InitialPreMMapHook), ""); 143 RAW_CHECK(MallocHook::RemovePreSbrkHook(&InitialPreSbrkHook), ""); 144 145 // HeapLeakChecker is currently the only module that needs to get control on 146 // the first memory allocation, but one can add other modules by following the 147 // same weak/strong function pattern. 148 MallocHook_InitAtFirstAllocation_HeapLeakChecker(); 149} 150 151} // namespace 152 153// Weak default initialization function that must go after its use. 154extern "C" void MallocHook_InitAtFirstAllocation_HeapLeakChecker() { 155 // Do nothing. 156} 157 158namespace base { namespace internal { 159 160// The code below is DEPRECATED. 161template<typename PtrT> 162PtrT AtomicPtr<PtrT>::Exchange(PtrT new_val) { 163 base::subtle::MemoryBarrier(); // Release semantics. 164 // Depending on the system, NoBarrier_AtomicExchange(AtomicWord*) 165 // may have been defined to return an AtomicWord, Atomic32, or 166 // Atomic64. We hide that implementation detail here with an 167 // explicit cast. This prevents MSVC 2005, at least, from complaining. 168 PtrT old_val = reinterpret_cast<PtrT>(static_cast<AtomicWord>( 169 base::subtle::NoBarrier_AtomicExchange( 170 &data_, 171 reinterpret_cast<AtomicWord>(new_val)))); 172 base::subtle::MemoryBarrier(); // And acquire semantics. 173 return old_val; 174} 175 176template<typename PtrT> 177PtrT AtomicPtr<PtrT>::CompareAndSwap(PtrT old_val, PtrT new_val) { 178 base::subtle::MemoryBarrier(); // Release semantics. 179 PtrT retval = reinterpret_cast<PtrT>(static_cast<AtomicWord>( 180 base::subtle::NoBarrier_CompareAndSwap( 181 &data_, 182 reinterpret_cast<AtomicWord>(old_val), 183 reinterpret_cast<AtomicWord>(new_val)))); 184 base::subtle::MemoryBarrier(); // And acquire semantics. 185 return retval; 186} 187 188AtomicPtr<MallocHook::NewHook> new_hook_ = { 0 }; 189AtomicPtr<MallocHook::DeleteHook> delete_hook_ = { 0 }; 190AtomicPtr<MallocHook::PreMmapHook> premmap_hook_ = { 0 }; 191AtomicPtr<MallocHook::MmapHook> mmap_hook_ = { 0 }; 192AtomicPtr<MallocHook::MunmapHook> munmap_hook_ = { 0 }; 193AtomicPtr<MallocHook::MremapHook> mremap_hook_ = { 0 }; 194AtomicPtr<MallocHook::PreSbrkHook> presbrk_hook_ = { 0 }; 195AtomicPtr<MallocHook::SbrkHook> sbrk_hook_ = { 0 }; 196// End of DEPRECATED code section. 197 198// This lock is shared between all implementations of HookList::Add & Remove. 199// The potential for contention is very small. This needs to be a SpinLock and 200// not a Mutex since it's possible for Mutex locking to allocate memory (e.g., 201// per-thread allocation in debug builds), which could cause infinite recursion. 202static SpinLock hooklist_spinlock(base::LINKER_INITIALIZED); 203 204template <typename T> 205bool HookList<T>::Add(T value_as_t) { 206 AtomicWord value = bit_cast<AtomicWord>(value_as_t); 207 if (value == 0) { 208 return false; 209 } 210 SpinLockHolder l(&hooklist_spinlock); 211 // Find the first slot in data that is 0. 212 int index = 0; 213 while ((index < kHookListMaxValues) && 214 (base::subtle::NoBarrier_Load(&priv_data[index]) != 0)) { 215 ++index; 216 } 217 if (index == kHookListMaxValues) { 218 return false; 219 } 220 AtomicWord prev_num_hooks = base::subtle::Acquire_Load(&priv_end); 221 base::subtle::Release_Store(&priv_data[index], value); 222 if (prev_num_hooks <= index) { 223 base::subtle::Release_Store(&priv_end, index + 1); 224 } 225 return true; 226} 227 228template <typename T> 229bool HookList<T>::Remove(T value_as_t) { 230 if (value_as_t == 0) { 231 return false; 232 } 233 SpinLockHolder l(&hooklist_spinlock); 234 AtomicWord hooks_end = base::subtle::Acquire_Load(&priv_end); 235 int index = 0; 236 while (index < hooks_end && value_as_t != bit_cast<T>( 237 base::subtle::Acquire_Load(&priv_data[index]))) { 238 ++index; 239 } 240 if (index == hooks_end) { 241 return false; 242 } 243 base::subtle::Release_Store(&priv_data[index], 0); 244 if (hooks_end == index + 1) { 245 // Adjust hooks_end down to the lowest possible value. 246 hooks_end = index; 247 while ((hooks_end > 0) && 248 (base::subtle::Acquire_Load(&priv_data[hooks_end - 1]) == 0)) { 249 --hooks_end; 250 } 251 base::subtle::Release_Store(&priv_end, hooks_end); 252 } 253 return true; 254} 255 256template <typename T> 257int HookList<T>::Traverse(T* output_array, int n) const { 258 AtomicWord hooks_end = base::subtle::Acquire_Load(&priv_end); 259 int actual_hooks_end = 0; 260 for (int i = 0; i < hooks_end && n > 0; ++i) { 261 AtomicWord data = base::subtle::Acquire_Load(&priv_data[i]); 262 if (data != 0) { 263 *output_array++ = bit_cast<T>(data); 264 ++actual_hooks_end; 265 --n; 266 } 267 } 268 return actual_hooks_end; 269} 270 271// Initialize a HookList (optionally with the given initial_value in index 0). 272#define INIT_HOOK_LIST { 0 } 273#define INIT_HOOK_LIST_WITH_VALUE(initial_value) \ 274 { 1, { reinterpret_cast<AtomicWord>(initial_value) } } 275 276// Explicit instantiation for malloc_hook_test.cc. This ensures all the methods 277// are instantiated. 278template struct HookList<MallocHook::NewHook>; 279 280HookList<MallocHook::NewHook> new_hooks_ = 281 INIT_HOOK_LIST_WITH_VALUE(&InitialNewHook); 282HookList<MallocHook::DeleteHook> delete_hooks_ = INIT_HOOK_LIST; 283HookList<MallocHook::PreMmapHook> premmap_hooks_ = 284 INIT_HOOK_LIST_WITH_VALUE(&InitialPreMMapHook); 285HookList<MallocHook::MmapHook> mmap_hooks_ = INIT_HOOK_LIST; 286HookList<MallocHook::MunmapHook> munmap_hooks_ = INIT_HOOK_LIST; 287HookList<MallocHook::MremapHook> mremap_hooks_ = INIT_HOOK_LIST; 288HookList<MallocHook::PreSbrkHook> presbrk_hooks_ = 289 INIT_HOOK_LIST_WITH_VALUE(InitialPreSbrkHook); 290HookList<MallocHook::SbrkHook> sbrk_hooks_ = INIT_HOOK_LIST; 291 292// These lists contain either 0 or 1 hooks. 293HookList<MallocHook::MmapReplacement> mmap_replacement_ = { 0 }; 294HookList<MallocHook::MunmapReplacement> munmap_replacement_ = { 0 }; 295 296#undef INIT_HOOK_LIST_WITH_VALUE 297#undef INIT_HOOK_LIST 298 299} } // namespace base::internal 300 301// The code below is DEPRECATED. 302using base::internal::new_hook_; 303using base::internal::delete_hook_; 304using base::internal::premmap_hook_; 305using base::internal::mmap_hook_; 306using base::internal::munmap_hook_; 307using base::internal::mremap_hook_; 308using base::internal::presbrk_hook_; 309using base::internal::sbrk_hook_; 310// End of DEPRECATED code section. 311 312using base::internal::kHookListMaxValues; 313using base::internal::new_hooks_; 314using base::internal::delete_hooks_; 315using base::internal::premmap_hooks_; 316using base::internal::mmap_hooks_; 317using base::internal::mmap_replacement_; 318using base::internal::munmap_hooks_; 319using base::internal::munmap_replacement_; 320using base::internal::mremap_hooks_; 321using base::internal::presbrk_hooks_; 322using base::internal::sbrk_hooks_; 323 324// These are available as C bindings as well as C++, hence their 325// definition outside the MallocHook class. 326extern "C" 327int MallocHook_AddNewHook(MallocHook_NewHook hook) { 328 RAW_VLOG(10, "AddNewHook(%p)", hook); 329 return new_hooks_.Add(hook); 330} 331 332extern "C" 333int MallocHook_RemoveNewHook(MallocHook_NewHook hook) { 334 RAW_VLOG(10, "RemoveNewHook(%p)", hook); 335 return new_hooks_.Remove(hook); 336} 337 338extern "C" 339int MallocHook_AddDeleteHook(MallocHook_DeleteHook hook) { 340 RAW_VLOG(10, "AddDeleteHook(%p)", hook); 341 return delete_hooks_.Add(hook); 342} 343 344extern "C" 345int MallocHook_RemoveDeleteHook(MallocHook_DeleteHook hook) { 346 RAW_VLOG(10, "RemoveDeleteHook(%p)", hook); 347 return delete_hooks_.Remove(hook); 348} 349 350extern "C" 351int MallocHook_AddPreMmapHook(MallocHook_PreMmapHook hook) { 352 RAW_VLOG(10, "AddPreMmapHook(%p)", hook); 353 return premmap_hooks_.Add(hook); 354} 355 356extern "C" 357int MallocHook_RemovePreMmapHook(MallocHook_PreMmapHook hook) { 358 RAW_VLOG(10, "RemovePreMmapHook(%p)", hook); 359 return premmap_hooks_.Remove(hook); 360} 361 362extern "C" 363int MallocHook_SetMmapReplacement(MallocHook_MmapReplacement hook) { 364 RAW_VLOG(10, "SetMmapReplacement(%p)", hook); 365 // NOTE this is a best effort CHECK. Concurrent sets could succeed since 366 // this test is outside of the Add spin lock. 367 RAW_CHECK(mmap_replacement_.empty(), "Only one MMapReplacement is allowed."); 368 return mmap_replacement_.Add(hook); 369} 370 371extern "C" 372int MallocHook_RemoveMmapReplacement(MallocHook_MmapReplacement hook) { 373 RAW_VLOG(10, "RemoveMmapReplacement(%p)", hook); 374 return mmap_replacement_.Remove(hook); 375} 376 377extern "C" 378int MallocHook_AddMmapHook(MallocHook_MmapHook hook) { 379 RAW_VLOG(10, "AddMmapHook(%p)", hook); 380 return mmap_hooks_.Add(hook); 381} 382 383extern "C" 384int MallocHook_RemoveMmapHook(MallocHook_MmapHook hook) { 385 RAW_VLOG(10, "RemoveMmapHook(%p)", hook); 386 return mmap_hooks_.Remove(hook); 387} 388 389extern "C" 390int MallocHook_AddMunmapHook(MallocHook_MunmapHook hook) { 391 RAW_VLOG(10, "AddMunmapHook(%p)", hook); 392 return munmap_hooks_.Add(hook); 393} 394 395extern "C" 396int MallocHook_RemoveMunmapHook(MallocHook_MunmapHook hook) { 397 RAW_VLOG(10, "RemoveMunmapHook(%p)", hook); 398 return munmap_hooks_.Remove(hook); 399} 400 401extern "C" 402int MallocHook_SetMunmapReplacement(MallocHook_MunmapReplacement hook) { 403 RAW_VLOG(10, "SetMunmapReplacement(%p)", hook); 404 // NOTE this is a best effort CHECK. Concurrent sets could succeed since 405 // this test is outside of the Add spin lock. 406 RAW_CHECK(munmap_replacement_.empty(), 407 "Only one MunmapReplacement is allowed."); 408 return munmap_replacement_.Add(hook); 409} 410 411extern "C" 412int MallocHook_RemoveMunmapReplacement(MallocHook_MunmapReplacement hook) { 413 RAW_VLOG(10, "RemoveMunmapReplacement(%p)", hook); 414 return munmap_replacement_.Remove(hook); 415} 416 417extern "C" 418int MallocHook_AddMremapHook(MallocHook_MremapHook hook) { 419 RAW_VLOG(10, "AddMremapHook(%p)", hook); 420 return mremap_hooks_.Add(hook); 421} 422 423extern "C" 424int MallocHook_RemoveMremapHook(MallocHook_MremapHook hook) { 425 RAW_VLOG(10, "RemoveMremapHook(%p)", hook); 426 return mremap_hooks_.Remove(hook); 427} 428 429extern "C" 430int MallocHook_AddPreSbrkHook(MallocHook_PreSbrkHook hook) { 431 RAW_VLOG(10, "AddPreSbrkHook(%p)", hook); 432 return presbrk_hooks_.Add(hook); 433} 434 435extern "C" 436int MallocHook_RemovePreSbrkHook(MallocHook_PreSbrkHook hook) { 437 RAW_VLOG(10, "RemovePreSbrkHook(%p)", hook); 438 return presbrk_hooks_.Remove(hook); 439} 440 441extern "C" 442int MallocHook_AddSbrkHook(MallocHook_SbrkHook hook) { 443 RAW_VLOG(10, "AddSbrkHook(%p)", hook); 444 return sbrk_hooks_.Add(hook); 445} 446 447extern "C" 448int MallocHook_RemoveSbrkHook(MallocHook_SbrkHook hook) { 449 RAW_VLOG(10, "RemoveSbrkHook(%p)", hook); 450 return sbrk_hooks_.Remove(hook); 451} 452 453// The code below is DEPRECATED. 454extern "C" 455MallocHook_NewHook MallocHook_SetNewHook(MallocHook_NewHook hook) { 456 RAW_VLOG(10, "SetNewHook(%p)", hook); 457 return new_hook_.Exchange(hook); 458} 459 460extern "C" 461MallocHook_DeleteHook MallocHook_SetDeleteHook(MallocHook_DeleteHook hook) { 462 RAW_VLOG(10, "SetDeleteHook(%p)", hook); 463 return delete_hook_.Exchange(hook); 464} 465 466extern "C" 467MallocHook_PreMmapHook MallocHook_SetPreMmapHook(MallocHook_PreMmapHook hook) { 468 RAW_VLOG(10, "SetPreMmapHook(%p)", hook); 469 return premmap_hook_.Exchange(hook); 470} 471 472extern "C" 473MallocHook_MmapHook MallocHook_SetMmapHook(MallocHook_MmapHook hook) { 474 RAW_VLOG(10, "SetMmapHook(%p)", hook); 475 return mmap_hook_.Exchange(hook); 476} 477 478extern "C" 479MallocHook_MunmapHook MallocHook_SetMunmapHook(MallocHook_MunmapHook hook) { 480 RAW_VLOG(10, "SetMunmapHook(%p)", hook); 481 return munmap_hook_.Exchange(hook); 482} 483 484extern "C" 485MallocHook_MremapHook MallocHook_SetMremapHook(MallocHook_MremapHook hook) { 486 RAW_VLOG(10, "SetMremapHook(%p)", hook); 487 return mremap_hook_.Exchange(hook); 488} 489 490extern "C" 491MallocHook_PreSbrkHook MallocHook_SetPreSbrkHook(MallocHook_PreSbrkHook hook) { 492 RAW_VLOG(10, "SetPreSbrkHook(%p)", hook); 493 return presbrk_hook_.Exchange(hook); 494} 495 496extern "C" 497MallocHook_SbrkHook MallocHook_SetSbrkHook(MallocHook_SbrkHook hook) { 498 RAW_VLOG(10, "SetSbrkHook(%p)", hook); 499 return sbrk_hook_.Exchange(hook); 500} 501// End of DEPRECATED code section. 502 503// Note: embedding the function calls inside the traversal of HookList would be 504// very confusing, as it is legal for a hook to remove itself and add other 505// hooks. Doing traversal first, and then calling the hooks ensures we only 506// call the hooks registered at the start. 507#define INVOKE_HOOKS(HookType, hook_list, args) do { \ 508 HookType hooks[kHookListMaxValues]; \ 509 int num_hooks = hook_list.Traverse(hooks, kHookListMaxValues); \ 510 for (int i = 0; i < num_hooks; ++i) { \ 511 (*hooks[i])args; \ 512 } \ 513 } while (0) 514 515// There should only be one replacement. Return the result of the first 516// one, or false if there is none. 517#define INVOKE_REPLACEMENT(HookType, hook_list, args) do { \ 518 HookType hooks[kHookListMaxValues]; \ 519 int num_hooks = hook_list.Traverse(hooks, kHookListMaxValues); \ 520 return (num_hooks > 0 && (*hooks[0])args); \ 521 } while (0) 522 523 524void MallocHook::InvokeNewHookSlow(const void* p, size_t s) { 525 INVOKE_HOOKS(NewHook, new_hooks_, (p, s)); 526} 527 528void MallocHook::InvokeDeleteHookSlow(const void* p) { 529 INVOKE_HOOKS(DeleteHook, delete_hooks_, (p)); 530} 531 532void MallocHook::InvokePreMmapHookSlow(const void* start, 533 size_t size, 534 int protection, 535 int flags, 536 int fd, 537 off_t offset) { 538 INVOKE_HOOKS(PreMmapHook, premmap_hooks_, (start, size, protection, flags, fd, 539 offset)); 540} 541 542void MallocHook::InvokeMmapHookSlow(const void* result, 543 const void* start, 544 size_t size, 545 int protection, 546 int flags, 547 int fd, 548 off_t offset) { 549 INVOKE_HOOKS(MmapHook, mmap_hooks_, (result, start, size, protection, flags, 550 fd, offset)); 551} 552 553bool MallocHook::InvokeMmapReplacementSlow(const void* start, 554 size_t size, 555 int protection, 556 int flags, 557 int fd, 558 off_t offset, 559 void** result) { 560 INVOKE_REPLACEMENT(MmapReplacement, mmap_replacement_, 561 (start, size, protection, flags, fd, offset, result)); 562} 563 564void MallocHook::InvokeMunmapHookSlow(const void* p, size_t s) { 565 INVOKE_HOOKS(MunmapHook, munmap_hooks_, (p, s)); 566} 567 568bool MallocHook::InvokeMunmapReplacementSlow(const void* p, 569 size_t s, 570 int* result) { 571 INVOKE_REPLACEMENT(MunmapReplacement, munmap_replacement_, (p, s, result)); 572} 573 574void MallocHook::InvokeMremapHookSlow(const void* result, 575 const void* old_addr, 576 size_t old_size, 577 size_t new_size, 578 int flags, 579 const void* new_addr) { 580 INVOKE_HOOKS(MremapHook, mremap_hooks_, (result, old_addr, old_size, new_size, 581 flags, new_addr)); 582} 583 584void MallocHook::InvokePreSbrkHookSlow(ptrdiff_t increment) { 585 INVOKE_HOOKS(PreSbrkHook, presbrk_hooks_, (increment)); 586} 587 588void MallocHook::InvokeSbrkHookSlow(const void* result, ptrdiff_t increment) { 589 INVOKE_HOOKS(SbrkHook, sbrk_hooks_, (result, increment)); 590} 591 592#undef INVOKE_HOOKS 593 594DEFINE_ATTRIBUTE_SECTION_VARS(google_malloc); 595DECLARE_ATTRIBUTE_SECTION_VARS(google_malloc); 596 // actual functions are in debugallocation.cc or tcmalloc.cc 597DEFINE_ATTRIBUTE_SECTION_VARS(malloc_hook); 598DECLARE_ATTRIBUTE_SECTION_VARS(malloc_hook); 599 // actual functions are in this file, malloc_hook.cc, and low_level_alloc.cc 600 601#define ADDR_IN_ATTRIBUTE_SECTION(addr, name) \ 602 (reinterpret_cast<uintptr_t>(ATTRIBUTE_SECTION_START(name)) <= \ 603 reinterpret_cast<uintptr_t>(addr) && \ 604 reinterpret_cast<uintptr_t>(addr) < \ 605 reinterpret_cast<uintptr_t>(ATTRIBUTE_SECTION_STOP(name))) 606 607// Return true iff 'caller' is a return address within a function 608// that calls one of our hooks via MallocHook:Invoke*. 609// A helper for GetCallerStackTrace. 610static inline bool InHookCaller(const void* caller) { 611 return ADDR_IN_ATTRIBUTE_SECTION(caller, google_malloc) || 612 ADDR_IN_ATTRIBUTE_SECTION(caller, malloc_hook); 613 // We can use one section for everything except tcmalloc_or_debug 614 // due to its special linkage mode, which prevents merging of the sections. 615} 616 617#undef ADDR_IN_ATTRIBUTE_SECTION 618 619static bool checked_sections = false; 620 621static inline void CheckInHookCaller() { 622 if (!checked_sections) { 623 INIT_ATTRIBUTE_SECTION_VARS(google_malloc); 624 if (ATTRIBUTE_SECTION_START(google_malloc) == 625 ATTRIBUTE_SECTION_STOP(google_malloc)) { 626 RAW_LOG(ERROR, "google_malloc section is missing, " 627 "thus InHookCaller is broken!"); 628 } 629 INIT_ATTRIBUTE_SECTION_VARS(malloc_hook); 630 if (ATTRIBUTE_SECTION_START(malloc_hook) == 631 ATTRIBUTE_SECTION_STOP(malloc_hook)) { 632 RAW_LOG(ERROR, "malloc_hook section is missing, " 633 "thus InHookCaller is broken!"); 634 } 635 checked_sections = true; 636 } 637} 638 639// We can improve behavior/compactness of this function 640// if we pass a generic test function (with a generic arg) 641// into the implementations for GetStackTrace instead of the skip_count. 642extern "C" int MallocHook_GetCallerStackTrace(void** result, int max_depth, 643 int skip_count) { 644#if defined(NO_TCMALLOC_SAMPLES) 645 return 0; 646#elif !defined(HAVE_ATTRIBUTE_SECTION_START) 647 // Fall back to GetStackTrace and good old but fragile frame skip counts. 648 // Note: this path is inaccurate when a hook is not called directly by an 649 // allocation function but is daisy-chained through another hook, 650 // search for MallocHook::(Get|Set|Invoke)* to find such cases. 651 return GetStackTrace(result, max_depth, skip_count + int(DEBUG_MODE)); 652 // due to -foptimize-sibling-calls in opt mode 653 // there's no need for extra frame skip here then 654#else 655 CheckInHookCaller(); 656 // MallocHook caller determination via InHookCaller works, use it: 657 static const int kMaxSkip = 32 + 6 + 3; 658 // Constant tuned to do just one GetStackTrace call below in practice 659 // and not get many frames that we don't actually need: 660 // currently max passsed max_depth is 32, 661 // max passed/needed skip_count is 6 662 // and 3 is to account for some hook daisy chaining. 663 static const int kStackSize = kMaxSkip + 1; 664 void* stack[kStackSize]; 665 int depth = GetStackTrace(stack, kStackSize, 1); // skip this function frame 666 if (depth == 0) // silenty propagate cases when GetStackTrace does not work 667 return 0; 668 for (int i = 0; i < depth; ++i) { // stack[0] is our immediate caller 669 if (InHookCaller(stack[i])) { 670 RAW_VLOG(10, "Found hooked allocator at %d: %p <- %p", 671 i, stack[i], stack[i+1]); 672 i += 1; // skip hook caller frame 673 depth -= i; // correct depth 674 if (depth > max_depth) depth = max_depth; 675 copy(stack + i, stack + i + depth, result); 676 if (depth < max_depth && depth + i == kStackSize) { 677 // get frames for the missing depth 678 depth += 679 GetStackTrace(result + depth, max_depth - depth, 1 + kStackSize); 680 } 681 return depth; 682 } 683 } 684 RAW_LOG(WARNING, "Hooked allocator frame not found, returning empty trace"); 685 // If this happens try increasing kMaxSkip 686 // or else something must be wrong with InHookCaller, 687 // e.g. for every section used in InHookCaller 688 // all functions in that section must be inside the same library. 689 return 0; 690#endif 691} 692 693// On systems where we know how, we override mmap/munmap/mremap/sbrk 694// to provide support for calling the related hooks (in addition, 695// of course, to doing what these functions normally do). 696 697#if defined(__linux) 698# include "malloc_hook_mmap_linux.h" 699 700#elif defined(__FreeBSD__) 701# include "malloc_hook_mmap_freebsd.h" 702 703#else 704 705/*static*/void* MallocHook::UnhookedMMap(void *start, size_t length, int prot, 706 int flags, int fd, off_t offset) { 707 void* result; 708 if (!MallocHook::InvokeMmapReplacement( 709 start, length, prot, flags, fd, offset, &result)) { 710 result = mmap(start, length, prot, flags, fd, offset); 711 } 712 return result; 713} 714 715/*static*/int MallocHook::UnhookedMUnmap(void *start, size_t length) { 716 int result; 717 if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) { 718 result = munmap(start, length); 719 } 720 return result; 721} 722 723#endif 724