1// Copyright (c) 2005, Google Inc. 2// All rights reserved. 3// 4// Redistribution and use in source and binary forms, with or without 5// modification, are permitted provided that the following conditions are 6// met: 7// 8// * Redistributions of source code must retain the above copyright 9// notice, this list of conditions and the following disclaimer. 10// * Redistributions in binary form must reproduce the above 11// copyright notice, this list of conditions and the following disclaimer 12// in the documentation and/or other materials provided with the 13// distribution. 14// * Neither the name of Google Inc. nor the names of its 15// contributors may be used to endorse or promote products derived from 16// this software without specific prior written permission. 17// 18// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30// --- 31// Author: Sanjay Ghemawat 32// 33// TODO: Log large allocations 34 35#include <config.h> 36#include <stddef.h> 37#include <stdio.h> 38#include <stdlib.h> 39#ifdef HAVE_UNISTD_H 40#include <unistd.h> 41#endif 42#ifdef HAVE_INTTYPES_H 43#include <inttypes.h> 44#endif 45#ifdef HAVE_FCNTL_H 46#include <fcntl.h> // for open() 47#endif 48#ifdef HAVE_MMAP 49#include <sys/mman.h> 50#endif 51#include <errno.h> 52#include <assert.h> 53#include <sys/types.h> 54 55#include <algorithm> 56#include <string> 57 58#include <gperftools/heap-profiler.h> 59 60#include "base/logging.h" 61#include "base/basictypes.h" // for PRId64, among other things 62#include "base/googleinit.h" 63#include "base/commandlineflags.h" 64#include "malloc_hook-inl.h" 65#include "tcmalloc_guard.h" 66#include <gperftools/malloc_hook.h> 67#include <gperftools/malloc_extension.h> 68#include "base/spinlock.h" 69#include "base/low_level_alloc.h" 70#include "base/sysinfo.h" // for GetUniquePathFromEnv() 71#include "heap-profile-table.h" 72#include "memory_region_map.h" 73 74 75#ifndef PATH_MAX 76#ifdef MAXPATHLEN 77#define PATH_MAX MAXPATHLEN 78#else 79#define PATH_MAX 4096 // seems conservative for max filename len! 80#endif 81#endif 82 83using STL_NAMESPACE::string; 84using STL_NAMESPACE::sort; 85 86//---------------------------------------------------------------------- 87// Flags that control heap-profiling 88// 89// The thread-safety of the profiler depends on these being immutable 90// after main starts, so don't change them. 91//---------------------------------------------------------------------- 92 93DEFINE_int64(heap_profile_allocation_interval, 94 EnvToInt64("HEAP_PROFILE_ALLOCATION_INTERVAL", 1 << 30 /*1GB*/), 95 "If non-zero, dump heap profiling information once every " 96 "specified number of bytes allocated by the program since " 97 "the last dump."); 98DEFINE_int64(heap_profile_deallocation_interval, 99 EnvToInt64("HEAP_PROFILE_DEALLOCATION_INTERVAL", 0), 100 "If non-zero, dump heap profiling information once every " 101 "specified number of bytes deallocated by the program " 102 "since the last dump."); 103// We could also add flags that report whenever inuse_bytes changes by 104// X or -X, but there hasn't been a need for that yet, so we haven't. 105DEFINE_int64(heap_profile_inuse_interval, 106 EnvToInt64("HEAP_PROFILE_INUSE_INTERVAL", 100 << 20 /*100MB*/), 107 "If non-zero, dump heap profiling information whenever " 108 "the high-water memory usage mark increases by the specified " 109 "number of bytes."); 110DEFINE_bool(mmap_log, 111 EnvToBool("HEAP_PROFILE_MMAP_LOG", false), 112 "Should mmap/munmap calls be logged?"); 113DEFINE_bool(mmap_profile, 114 EnvToBool("HEAP_PROFILE_MMAP", false), 115 "If heap-profiling is on, also profile mmap, mremap, and sbrk)"); 116DEFINE_bool(only_mmap_profile, 117 EnvToBool("HEAP_PROFILE_ONLY_MMAP", false), 118 "If heap-profiling is on, only profile mmap, mremap, and sbrk; " 119 "do not profile malloc/new/etc"); 120 121 122//---------------------------------------------------------------------- 123// Locking 124//---------------------------------------------------------------------- 125 126// A pthread_mutex has way too much lock contention to be used here. 127// 128// I would like to use Mutex, but it can call malloc(), 129// which can cause us to fall into an infinite recursion. 130// 131// So we use a simple spinlock. 132static SpinLock heap_lock(SpinLock::LINKER_INITIALIZED); 133 134//---------------------------------------------------------------------- 135// Simple allocator for heap profiler's internal memory 136//---------------------------------------------------------------------- 137 138static LowLevelAlloc::Arena *heap_profiler_memory; 139 140static void* ProfilerMalloc(size_t bytes) { 141 return LowLevelAlloc::AllocWithArena(bytes, heap_profiler_memory); 142} 143static void ProfilerFree(void* p) { 144 LowLevelAlloc::Free(p); 145} 146 147// We use buffers of this size in DoGetHeapProfile. 148static const int kProfileBufferSize = 1 << 20; 149 150// This is a last-ditch buffer we use in DumpProfileLocked in case we 151// can't allocate more memory from ProfilerMalloc. We expect this 152// will be used by HeapProfileEndWriter when the application has to 153// exit due to out-of-memory. This buffer is allocated in 154// HeapProfilerStart. Access to this must be protected by heap_lock. 155static char* global_profiler_buffer = NULL; 156 157 158//---------------------------------------------------------------------- 159// Profiling control/state data 160//---------------------------------------------------------------------- 161 162// Access to all of these is protected by heap_lock. 163static bool is_on = false; // If are on as a subsytem. 164static bool dumping = false; // Dumping status to prevent recursion 165static char* filename_prefix = NULL; // Prefix used for profile file names 166 // (NULL if no need for dumping yet) 167static int dump_count = 0; // How many dumps so far 168static int64 last_dump_alloc = 0; // alloc_size when did we last dump 169static int64 last_dump_free = 0; // free_size when did we last dump 170static int64 high_water_mark = 0; // In-use-bytes at last high-water dump 171 172static HeapProfileTable* heap_profile = NULL; // the heap profile table 173 174//---------------------------------------------------------------------- 175// Profile generation 176//---------------------------------------------------------------------- 177 178// Input must be a buffer of size at least 1MB. 179static char* DoGetHeapProfileLocked(char* buf, int buflen) { 180 // We used to be smarter about estimating the required memory and 181 // then capping it to 1MB and generating the profile into that. 182 if (buf == NULL || buflen < 1) 183 return NULL; 184 185 RAW_DCHECK(heap_lock.IsHeld(), ""); 186 int bytes_written = 0; 187 if (is_on) { 188 if (FLAGS_mmap_profile) { 189 heap_profile->RefreshMMapData(); 190 } 191 bytes_written = heap_profile->FillOrderedProfile(buf, buflen - 1); 192 if (FLAGS_mmap_profile) { 193 heap_profile->ClearMMapData(); 194 } 195 } 196 buf[bytes_written] = '\0'; 197 RAW_DCHECK(bytes_written == strlen(buf), ""); 198 199 return buf; 200} 201 202extern "C" char* GetHeapProfile() { 203 // Use normal malloc: we return the profile to the user to free it: 204 char* buffer = reinterpret_cast<char*>(malloc(kProfileBufferSize)); 205 SpinLockHolder l(&heap_lock); 206 return DoGetHeapProfileLocked(buffer, kProfileBufferSize); 207} 208 209// defined below 210static void NewHook(const void* ptr, size_t size); 211static void DeleteHook(const void* ptr); 212 213// Helper for HeapProfilerDump. 214static void DumpProfileLocked(const char* reason) { 215 RAW_DCHECK(heap_lock.IsHeld(), ""); 216 RAW_DCHECK(is_on, ""); 217 RAW_DCHECK(!dumping, ""); 218 219 if (filename_prefix == NULL) return; // we do not yet need dumping 220 221 dumping = true; 222 223 // Make file name 224 char file_name[1000]; 225 dump_count++; 226 snprintf(file_name, sizeof(file_name), "%s.%04d%s", 227 filename_prefix, dump_count, HeapProfileTable::kFileExt); 228 229 // Dump the profile 230 RAW_VLOG(0, "Dumping heap profile to %s (%s)", file_name, reason); 231 // We must use file routines that don't access memory, since we hold 232 // a memory lock now. 233 RawFD fd = RawOpenForWriting(file_name); 234 if (fd == kIllegalRawFD) { 235 RAW_LOG(ERROR, "Failed dumping heap profile to %s", file_name); 236 dumping = false; 237 return; 238 } 239 240 // This case may be impossible, but it's best to be safe. 241 // It's safe to use the global buffer: we're protected by heap_lock. 242 if (global_profiler_buffer == NULL) { 243 global_profiler_buffer = 244 reinterpret_cast<char*>(ProfilerMalloc(kProfileBufferSize)); 245 } 246 247 char* profile = DoGetHeapProfileLocked(global_profiler_buffer, 248 kProfileBufferSize); 249 RawWrite(fd, profile, strlen(profile)); 250 RawClose(fd); 251 252 dumping = false; 253} 254 255//---------------------------------------------------------------------- 256// Profile collection 257//---------------------------------------------------------------------- 258 259// Dump a profile after either an allocation or deallocation, if 260// the memory use has changed enough since the last dump. 261static void MaybeDumpProfileLocked() { 262 if (!dumping) { 263 const HeapProfileTable::Stats& total = heap_profile->total(); 264 const int64 inuse_bytes = total.alloc_size - total.free_size; 265 bool need_to_dump = false; 266 char buf[128]; 267 if (FLAGS_heap_profile_allocation_interval > 0 && 268 total.alloc_size >= 269 last_dump_alloc + FLAGS_heap_profile_allocation_interval) { 270 snprintf(buf, sizeof(buf), ("%"PRId64" MB allocated cumulatively, " 271 "%"PRId64" MB currently in use"), 272 total.alloc_size >> 20, inuse_bytes >> 20); 273 need_to_dump = true; 274 } else if (FLAGS_heap_profile_deallocation_interval > 0 && 275 total.free_size >= 276 last_dump_free + FLAGS_heap_profile_deallocation_interval) { 277 snprintf(buf, sizeof(buf), ("%"PRId64" MB freed cumulatively, " 278 "%"PRId64" MB currently in use"), 279 total.free_size >> 20, inuse_bytes >> 20); 280 need_to_dump = true; 281 } else if (FLAGS_heap_profile_inuse_interval > 0 && 282 inuse_bytes > 283 high_water_mark + FLAGS_heap_profile_inuse_interval) { 284 snprintf(buf, sizeof(buf), "%"PRId64" MB currently in use", 285 inuse_bytes >> 20); 286 need_to_dump = true; 287 } 288 if (need_to_dump) { 289 DumpProfileLocked(buf); 290 291 last_dump_alloc = total.alloc_size; 292 last_dump_free = total.free_size; 293 if (inuse_bytes > high_water_mark) 294 high_water_mark = inuse_bytes; 295 } 296 } 297} 298 299// Record an allocation in the profile. 300static void RecordAlloc(const void* ptr, size_t bytes, int skip_count) { 301 // Take the stack trace outside the critical section. 302 void* stack[HeapProfileTable::kMaxStackDepth]; 303 int depth = HeapProfileTable::GetCallerStackTrace(skip_count + 1, stack); 304 SpinLockHolder l(&heap_lock); 305 if (is_on) { 306 heap_profile->RecordAlloc(ptr, bytes, depth, stack); 307 MaybeDumpProfileLocked(); 308 } 309} 310 311// Record a deallocation in the profile. 312static void RecordFree(const void* ptr) { 313 SpinLockHolder l(&heap_lock); 314 if (is_on) { 315 heap_profile->RecordFree(ptr); 316 MaybeDumpProfileLocked(); 317 } 318} 319 320//---------------------------------------------------------------------- 321// Allocation/deallocation hooks for MallocHook 322//---------------------------------------------------------------------- 323 324// static 325void NewHook(const void* ptr, size_t size) { 326 if (ptr != NULL) RecordAlloc(ptr, size, 0); 327} 328 329// static 330void DeleteHook(const void* ptr) { 331 if (ptr != NULL) RecordFree(ptr); 332} 333 334// TODO(jandrews): Re-enable stack tracing 335#ifdef TODO_REENABLE_STACK_TRACING 336static void RawInfoStackDumper(const char* message, void*) { 337 RAW_LOG(INFO, "%.*s", static_cast<int>(strlen(message) - 1), message); 338 // -1 is to chop the \n which will be added by RAW_LOG 339} 340#endif 341 342static void MmapHook(const void* result, const void* start, size_t size, 343 int prot, int flags, int fd, off_t offset) { 344 if (FLAGS_mmap_log) { // log it 345 // We use PRIxS not just '%p' to avoid deadlocks 346 // in pretty-printing of NULL as "nil". 347 // TODO(maxim): instead should use a safe snprintf reimplementation 348 RAW_LOG(INFO, 349 "mmap(start=0x%"PRIxPTR", len=%"PRIuS", prot=0x%x, flags=0x%x, " 350 "fd=%d, offset=0x%x) = 0x%"PRIxPTR"", 351 (uintptr_t) start, size, prot, flags, fd, (unsigned int) offset, 352 (uintptr_t) result); 353#ifdef TODO_REENABLE_STACK_TRACING 354 DumpStackTrace(1, RawInfoStackDumper, NULL); 355#endif 356 } 357} 358 359static void MremapHook(const void* result, const void* old_addr, 360 size_t old_size, size_t new_size, 361 int flags, const void* new_addr) { 362 if (FLAGS_mmap_log) { // log it 363 // We use PRIxS not just '%p' to avoid deadlocks 364 // in pretty-printing of NULL as "nil". 365 // TODO(maxim): instead should use a safe snprintf reimplementation 366 RAW_LOG(INFO, 367 "mremap(old_addr=0x%"PRIxPTR", old_size=%"PRIuS", " 368 "new_size=%"PRIuS", flags=0x%x, new_addr=0x%"PRIxPTR") = " 369 "0x%"PRIxPTR"", 370 (uintptr_t) old_addr, old_size, new_size, flags, 371 (uintptr_t) new_addr, (uintptr_t) result); 372#ifdef TODO_REENABLE_STACK_TRACING 373 DumpStackTrace(1, RawInfoStackDumper, NULL); 374#endif 375 } 376} 377 378static void MunmapHook(const void* ptr, size_t size) { 379 if (FLAGS_mmap_log) { // log it 380 // We use PRIxS not just '%p' to avoid deadlocks 381 // in pretty-printing of NULL as "nil". 382 // TODO(maxim): instead should use a safe snprintf reimplementation 383 RAW_LOG(INFO, "munmap(start=0x%"PRIxPTR", len=%"PRIuS")", 384 (uintptr_t) ptr, size); 385#ifdef TODO_REENABLE_STACK_TRACING 386 DumpStackTrace(1, RawInfoStackDumper, NULL); 387#endif 388 } 389} 390 391static void SbrkHook(const void* result, ptrdiff_t increment) { 392 if (FLAGS_mmap_log) { // log it 393 RAW_LOG(INFO, "sbrk(inc=%"PRIdS") = 0x%"PRIxPTR"", 394 increment, (uintptr_t) result); 395#ifdef TODO_REENABLE_STACK_TRACING 396 DumpStackTrace(1, RawInfoStackDumper, NULL); 397#endif 398 } 399} 400 401//---------------------------------------------------------------------- 402// Starting/stopping/dumping 403//---------------------------------------------------------------------- 404 405extern "C" void HeapProfilerStart(const char* prefix) { 406 SpinLockHolder l(&heap_lock); 407 408 if (is_on) return; 409 410 is_on = true; 411 412 RAW_VLOG(0, "Starting tracking the heap"); 413 414 // This should be done before the hooks are set up, since it should 415 // call new, and we want that to be accounted for correctly. 416 MallocExtension::Initialize(); 417 418 if (FLAGS_only_mmap_profile) { 419 FLAGS_mmap_profile = true; 420 } 421 422 if (FLAGS_mmap_profile) { 423 // Ask MemoryRegionMap to record all mmap, mremap, and sbrk 424 // call stack traces of at least size kMaxStackDepth: 425 MemoryRegionMap::Init(HeapProfileTable::kMaxStackDepth); 426 } 427 428 if (FLAGS_mmap_log) { 429 // Install our hooks to do the logging: 430 RAW_CHECK(MallocHook::AddMmapHook(&MmapHook), ""); 431 RAW_CHECK(MallocHook::AddMremapHook(&MremapHook), ""); 432 RAW_CHECK(MallocHook::AddMunmapHook(&MunmapHook), ""); 433 RAW_CHECK(MallocHook::AddSbrkHook(&SbrkHook), ""); 434 } 435 436 heap_profiler_memory = 437 LowLevelAlloc::NewArena(0, LowLevelAlloc::DefaultArena()); 438 439 // Reserve space now for the heap profiler, so we can still write a 440 // heap profile even if the application runs out of memory. 441 global_profiler_buffer = 442 reinterpret_cast<char*>(ProfilerMalloc(kProfileBufferSize)); 443 444 heap_profile = new(ProfilerMalloc(sizeof(HeapProfileTable))) 445 HeapProfileTable(ProfilerMalloc, ProfilerFree); 446 447 last_dump_alloc = 0; 448 last_dump_free = 0; 449 high_water_mark = 0; 450 451 // We do not reset dump_count so if the user does a sequence of 452 // HeapProfilerStart/HeapProfileStop, we will get a continuous 453 // sequence of profiles. 454 455 if (FLAGS_only_mmap_profile == false) { 456 // Now set the hooks that capture new/delete and malloc/free. 457 RAW_CHECK(MallocHook::AddNewHook(&NewHook), ""); 458 RAW_CHECK(MallocHook::AddDeleteHook(&DeleteHook), ""); 459 } 460 461 // Copy filename prefix 462 RAW_DCHECK(filename_prefix == NULL, ""); 463 const int prefix_length = strlen(prefix); 464 filename_prefix = reinterpret_cast<char*>(ProfilerMalloc(prefix_length + 1)); 465 memcpy(filename_prefix, prefix, prefix_length); 466 filename_prefix[prefix_length] = '\0'; 467} 468 469extern "C" int IsHeapProfilerRunning() { 470 SpinLockHolder l(&heap_lock); 471 return is_on ? 1 : 0; // return an int, because C code doesn't have bool 472} 473 474extern "C" void HeapProfilerStop() { 475 SpinLockHolder l(&heap_lock); 476 477 if (!is_on) return; 478 479 if (FLAGS_only_mmap_profile == false) { 480 // Unset our new/delete hooks, checking they were set: 481 RAW_CHECK(MallocHook::RemoveNewHook(&NewHook), ""); 482 RAW_CHECK(MallocHook::RemoveDeleteHook(&DeleteHook), ""); 483 } 484 if (FLAGS_mmap_log) { 485 // Restore mmap/sbrk hooks, checking that our hooks were set: 486 RAW_CHECK(MallocHook::RemoveMmapHook(&MmapHook), ""); 487 RAW_CHECK(MallocHook::RemoveMremapHook(&MremapHook), ""); 488 RAW_CHECK(MallocHook::RemoveSbrkHook(&SbrkHook), ""); 489 RAW_CHECK(MallocHook::RemoveMunmapHook(&MunmapHook), ""); 490 } 491 492 // free profile 493 heap_profile->~HeapProfileTable(); 494 ProfilerFree(heap_profile); 495 heap_profile = NULL; 496 497 // free output-buffer memory 498 ProfilerFree(global_profiler_buffer); 499 500 // free prefix 501 ProfilerFree(filename_prefix); 502 filename_prefix = NULL; 503 504 if (!LowLevelAlloc::DeleteArena(heap_profiler_memory)) { 505 RAW_LOG(FATAL, "Memory leak in HeapProfiler:"); 506 } 507 508 if (FLAGS_mmap_profile) { 509 MemoryRegionMap::Shutdown(); 510 } 511 512 is_on = false; 513} 514 515extern "C" void HeapProfilerDump(const char *reason) { 516 SpinLockHolder l(&heap_lock); 517 if (is_on && !dumping) { 518 DumpProfileLocked(reason); 519 } 520} 521 522//---------------------------------------------------------------------- 523// Initialization/finalization code 524//---------------------------------------------------------------------- 525 526// Initialization code 527static void HeapProfilerInit() { 528 // Everything after this point is for setting up the profiler based on envvar 529 char fname[PATH_MAX]; 530 if (!GetUniquePathFromEnv("HEAPPROFILE", fname)) { 531 return; 532 } 533 // We do a uid check so we don't write out files in a setuid executable. 534#ifdef HAVE_GETEUID 535 if (getuid() != geteuid()) { 536 RAW_LOG(WARNING, ("HeapProfiler: ignoring HEAPPROFILE because " 537 "program seems to be setuid\n")); 538 return; 539 } 540#endif 541 542 HeapProfileTable::CleanupOldProfiles(fname); 543 544 HeapProfilerStart(fname); 545} 546 547// class used for finalization -- dumps the heap-profile at program exit 548struct HeapProfileEndWriter { 549 ~HeapProfileEndWriter() { HeapProfilerDump("Exiting"); } 550}; 551 552// We want to make sure tcmalloc is up and running before starting the profiler 553static const TCMallocGuard tcmalloc_initializer; 554REGISTER_MODULE_INITIALIZER(heapprofiler, HeapProfilerInit()); 555static HeapProfileEndWriter heap_profile_end_writer; 556