1c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev//=-- lsan_allocator.cc ---------------------------------------------------===// 2c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev// 3c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev// The LLVM Compiler Infrastructure 4c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev// 5c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev// This file is distributed under the University of Illinois Open Source 6c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev// License. See LICENSE.TXT for details. 7c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev// 8c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev//===----------------------------------------------------------------------===// 9c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev// 10c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev// This file is a part of LeakSanitizer. 11c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev// See lsan_allocator.h for details. 12c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev// 13c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev//===----------------------------------------------------------------------===// 14c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 15c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev#include "lsan_allocator.h" 16c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 17c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev#include "sanitizer_common/sanitizer_allocator.h" 18c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev#include "sanitizer_common/sanitizer_internal_defs.h" 19c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev#include "sanitizer_common/sanitizer_stackdepot.h" 20c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev#include "sanitizer_common/sanitizer_stacktrace.h" 21c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev#include "lsan_common.h" 22c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 2312f610efe6f48efb2d03b985b3b2a36d4556fca7Sergey Matveevextern "C" void *memset(void *ptr, int value, uptr num); 2412f610efe6f48efb2d03b985b3b2a36d4556fca7Sergey Matveev 25c99de512472c5635e554d983c225aa3a92a20202Sergey Matveevnamespace __lsan { 26c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 27ef89d6b576bd8f5084b70e2d908cd53c37fab9f9Sergey Matveevstatic const uptr kMaxAllowedMallocSize = 8UL << 30; 28c99de512472c5635e554d983c225aa3a92a20202Sergey Matveevstatic const uptr kAllocatorSpace = 0x600000000000ULL; 29ef89d6b576bd8f5084b70e2d908cd53c37fab9f9Sergey Matveevstatic const uptr kAllocatorSize = 0x40000000000ULL; // 4T. 30c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 31c99de512472c5635e554d983c225aa3a92a20202Sergey Matveevstruct ChunkMetadata { 32c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev bool allocated : 8; // Must be first. 33c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev ChunkTag tag : 2; 34c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev uptr requested_size : 54; 35c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev u32 stack_trace_id; 36c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev}; 37c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 38c99de512472c5635e554d983c225aa3a92a20202Sergey Matveevtypedef SizeClassAllocator64<kAllocatorSpace, kAllocatorSize, 3912f610efe6f48efb2d03b985b3b2a36d4556fca7Sergey Matveev sizeof(ChunkMetadata), DefaultSizeClassMap> PrimaryAllocator; 40c99de512472c5635e554d983c225aa3a92a20202Sergey Matveevtypedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; 41c99de512472c5635e554d983c225aa3a92a20202Sergey Matveevtypedef LargeMmapAllocator<> SecondaryAllocator; 42c99de512472c5635e554d983c225aa3a92a20202Sergey Matveevtypedef CombinedAllocator<PrimaryAllocator, AllocatorCache, 43c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev SecondaryAllocator> Allocator; 44c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 45c99de512472c5635e554d983c225aa3a92a20202Sergey Matveevstatic Allocator allocator; 46c99de512472c5635e554d983c225aa3a92a20202Sergey Matveevstatic THREADLOCAL AllocatorCache cache; 47c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 48c99de512472c5635e554d983c225aa3a92a20202Sergey Matveevvoid InitializeAllocator() { 49c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev allocator.Init(); 50c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev} 51c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 52c99de512472c5635e554d983c225aa3a92a20202Sergey Matveevvoid AllocatorThreadFinish() { 53c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev allocator.SwallowCache(&cache); 54c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev} 55c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 56c99de512472c5635e554d983c225aa3a92a20202Sergey Matveevstatic ChunkMetadata *Metadata(void *p) { 57ac78d0087281e9b40bb043f827fbde8c05c6505aSergey Matveev return reinterpret_cast<ChunkMetadata *>(allocator.GetMetaData(p)); 58c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev} 59c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 60c99de512472c5635e554d983c225aa3a92a20202Sergey Matveevstatic void RegisterAllocation(const StackTrace &stack, void *p, uptr size) { 61c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev if (!p) return; 62c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev ChunkMetadata *m = Metadata(p); 63c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev CHECK(m); 64200afbd8ba4904241c1ebcef4fa79d739ca01f73Sergey Matveev m->tag = DisabledInThisThread() ? kIgnored : kDirectlyLeaked; 65c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev m->stack_trace_id = StackDepotPut(stack.trace, stack.size); 66c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev m->requested_size = size; 67ac78d0087281e9b40bb043f827fbde8c05c6505aSergey Matveev atomic_store(reinterpret_cast<atomic_uint8_t *>(m), 1, memory_order_relaxed); 68c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev} 69c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 70c99de512472c5635e554d983c225aa3a92a20202Sergey Matveevstatic void RegisterDeallocation(void *p) { 71c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev if (!p) return; 72c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev ChunkMetadata *m = Metadata(p); 73c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev CHECK(m); 74ac78d0087281e9b40bb043f827fbde8c05c6505aSergey Matveev atomic_store(reinterpret_cast<atomic_uint8_t *>(m), 0, memory_order_relaxed); 75c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev} 76c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 77c99de512472c5635e554d983c225aa3a92a20202Sergey Matveevvoid *Allocate(const StackTrace &stack, uptr size, uptr alignment, 78c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev bool cleared) { 79c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev if (size == 0) 80c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev size = 1; 81c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev if (size > kMaxAllowedMallocSize) { 82ef89d6b576bd8f5084b70e2d908cd53c37fab9f9Sergey Matveev Report("WARNING: LeakSanitizer failed to allocate %zu bytes\n", size); 83ef89d6b576bd8f5084b70e2d908cd53c37fab9f9Sergey Matveev return 0; 84c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev } 8512f610efe6f48efb2d03b985b3b2a36d4556fca7Sergey Matveev void *p = allocator.Allocate(&cache, size, alignment, false); 8612f610efe6f48efb2d03b985b3b2a36d4556fca7Sergey Matveev // Do not rely on the allocator to clear the memory (it's slow). 8712f610efe6f48efb2d03b985b3b2a36d4556fca7Sergey Matveev if (cleared && allocator.FromPrimary(p)) 8812f610efe6f48efb2d03b985b3b2a36d4556fca7Sergey Matveev memset(p, 0, size); 89c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev RegisterAllocation(stack, p, size); 90c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev return p; 91c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev} 92c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 93c99de512472c5635e554d983c225aa3a92a20202Sergey Matveevvoid Deallocate(void *p) { 94c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev RegisterDeallocation(p); 95c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev allocator.Deallocate(&cache, p); 96c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev} 97c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 98c99de512472c5635e554d983c225aa3a92a20202Sergey Matveevvoid *Reallocate(const StackTrace &stack, void *p, uptr new_size, 99c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev uptr alignment) { 100c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev RegisterDeallocation(p); 101c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev if (new_size > kMaxAllowedMallocSize) { 102ef89d6b576bd8f5084b70e2d908cd53c37fab9f9Sergey Matveev Report("WARNING: LeakSanitizer failed to allocate %zu bytes\n", new_size); 103ef89d6b576bd8f5084b70e2d908cd53c37fab9f9Sergey Matveev allocator.Deallocate(&cache, p); 104ef89d6b576bd8f5084b70e2d908cd53c37fab9f9Sergey Matveev return 0; 105c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev } 106c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev p = allocator.Reallocate(&cache, p, new_size, alignment); 107c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev RegisterAllocation(stack, p, new_size); 108c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev return p; 109c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev} 110c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 111c99de512472c5635e554d983c225aa3a92a20202Sergey Matveevvoid GetAllocatorCacheRange(uptr *begin, uptr *end) { 112c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev *begin = (uptr)&cache; 113c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev *end = *begin + sizeof(cache); 114c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev} 115c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 116c99de512472c5635e554d983c225aa3a92a20202Sergey Matveevuptr GetMallocUsableSize(void *p) { 117c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev ChunkMetadata *m = Metadata(p); 118c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev if (!m) return 0; 119c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev return m->requested_size; 120c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev} 121c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 122c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev///// Interface to the common LSan module. ///// 123c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 124c99de512472c5635e554d983c225aa3a92a20202Sergey Matveevvoid LockAllocator() { 125c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev allocator.ForceLock(); 126c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev} 127c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 128c99de512472c5635e554d983c225aa3a92a20202Sergey Matveevvoid UnlockAllocator() { 129c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev allocator.ForceUnlock(); 130c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev} 131c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 132c99de512472c5635e554d983c225aa3a92a20202Sergey Matveevvoid GetAllocatorGlobalRange(uptr *begin, uptr *end) { 133c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev *begin = (uptr)&allocator; 134c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev *end = *begin + sizeof(allocator); 135c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev} 136c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 137ac78d0087281e9b40bb043f827fbde8c05c6505aSergey Matveevuptr PointsIntoChunk(void* p) { 138ac78d0087281e9b40bb043f827fbde8c05c6505aSergey Matveev uptr addr = reinterpret_cast<uptr>(p); 139ac78d0087281e9b40bb043f827fbde8c05c6505aSergey Matveev uptr chunk = reinterpret_cast<uptr>(allocator.GetBlockBeginFastLocked(p)); 140c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev if (!chunk) return 0; 141c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev // LargeMmapAllocator considers pointers to the meta-region of a chunk to be 142c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev // valid, but we don't want that. 143ac78d0087281e9b40bb043f827fbde8c05c6505aSergey Matveev if (addr < chunk) return 0; 144ac78d0087281e9b40bb043f827fbde8c05c6505aSergey Matveev ChunkMetadata *m = Metadata(reinterpret_cast<void *>(chunk)); 145c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev CHECK(m); 1462d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (!m->allocated) 1472d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return 0; 1482d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (addr < chunk + m->requested_size) 1492d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return chunk; 1502d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (IsSpecialCaseOfOperatorNew0(chunk, m->requested_size, addr)) 151c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev return chunk; 152c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev return 0; 153c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev} 154c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 155ac78d0087281e9b40bb043f827fbde8c05c6505aSergey Matveevuptr GetUserBegin(uptr chunk) { 156ac78d0087281e9b40bb043f827fbde8c05c6505aSergey Matveev return chunk; 15729b756851c04df07f1584dfd77021dd2f37a7391Sergey Matveev} 15829b756851c04df07f1584dfd77021dd2f37a7391Sergey Matveev 159ac78d0087281e9b40bb043f827fbde8c05c6505aSergey MatveevLsanMetadata::LsanMetadata(uptr chunk) { 160ac78d0087281e9b40bb043f827fbde8c05c6505aSergey Matveev metadata_ = Metadata(reinterpret_cast<void *>(chunk)); 161c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev CHECK(metadata_); 162c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev} 163c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 164c99de512472c5635e554d983c225aa3a92a20202Sergey Matveevbool LsanMetadata::allocated() const { 165c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev return reinterpret_cast<ChunkMetadata *>(metadata_)->allocated; 166c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev} 167c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 168c99de512472c5635e554d983c225aa3a92a20202Sergey MatveevChunkTag LsanMetadata::tag() const { 169c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev return reinterpret_cast<ChunkMetadata *>(metadata_)->tag; 170c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev} 171c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 172c99de512472c5635e554d983c225aa3a92a20202Sergey Matveevvoid LsanMetadata::set_tag(ChunkTag value) { 173c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev reinterpret_cast<ChunkMetadata *>(metadata_)->tag = value; 174c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev} 175c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 176c99de512472c5635e554d983c225aa3a92a20202Sergey Matveevuptr LsanMetadata::requested_size() const { 177c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev return reinterpret_cast<ChunkMetadata *>(metadata_)->requested_size; 178c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev} 179c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 180c99de512472c5635e554d983c225aa3a92a20202Sergey Matveevu32 LsanMetadata::stack_trace_id() const { 181c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev return reinterpret_cast<ChunkMetadata *>(metadata_)->stack_trace_id; 182c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev} 183c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 184ac78d0087281e9b40bb043f827fbde8c05c6505aSergey Matveevvoid ForEachChunk(ForEachChunkCallback callback, void *arg) { 185ac78d0087281e9b40bb043f827fbde8c05c6505aSergey Matveev allocator.ForEachChunk(callback, arg); 186c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev} 187c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev 188cd571e07fd1179383188c70338fa0dc1c452cb19Sergey MatveevIgnoreObjectResult IgnoreObjectLocked(const void *p) { 189cd571e07fd1179383188c70338fa0dc1c452cb19Sergey Matveev void *chunk = allocator.GetBlockBegin(p); 190cd571e07fd1179383188c70338fa0dc1c452cb19Sergey Matveev if (!chunk || p < chunk) return kIgnoreObjectInvalid; 191cd571e07fd1179383188c70338fa0dc1c452cb19Sergey Matveev ChunkMetadata *m = Metadata(chunk); 192cd571e07fd1179383188c70338fa0dc1c452cb19Sergey Matveev CHECK(m); 193cd571e07fd1179383188c70338fa0dc1c452cb19Sergey Matveev if (m->allocated && (uptr)p < (uptr)chunk + m->requested_size) { 194b3b46dad13a2111a51fb1a67f36c8b633410e9b7Sergey Matveev if (m->tag == kIgnored) 195cd571e07fd1179383188c70338fa0dc1c452cb19Sergey Matveev return kIgnoreObjectAlreadyIgnored; 196b3b46dad13a2111a51fb1a67f36c8b633410e9b7Sergey Matveev m->tag = kIgnored; 197cd571e07fd1179383188c70338fa0dc1c452cb19Sergey Matveev return kIgnoreObjectSuccess; 198cd571e07fd1179383188c70338fa0dc1c452cb19Sergey Matveev } else { 199cd571e07fd1179383188c70338fa0dc1c452cb19Sergey Matveev return kIgnoreObjectInvalid; 200cd571e07fd1179383188c70338fa0dc1c452cb19Sergey Matveev } 201cd571e07fd1179383188c70338fa0dc1c452cb19Sergey Matveev} 202c99de512472c5635e554d983c225aa3a92a20202Sergey Matveev} // namespace __lsan 203