asan_allocator2.cc revision 5e719a705666988781b9735d62cafc808ade60e2
1//===-- asan_allocator2.cc ------------------------------------------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file is a part of AddressSanitizer, an address sanity checker. 11// 12// Implementation of ASan's memory allocator, 2-nd version. 13// This variant uses the allocator from sanitizer_common, i.e. the one shared 14// with ThreadSanitizer and MemorySanitizer. 15// 16//===----------------------------------------------------------------------===// 17#include "asan_allocator.h" 18 19#include "asan_mapping.h" 20#include "asan_poisoning.h" 21#include "asan_report.h" 22#include "asan_thread.h" 23#include "sanitizer_common/sanitizer_allocator.h" 24#include "sanitizer_common/sanitizer_flags.h" 25#include "sanitizer_common/sanitizer_internal_defs.h" 26#include "sanitizer_common/sanitizer_list.h" 27#include "sanitizer_common/sanitizer_stackdepot.h" 28#include "sanitizer_common/sanitizer_quarantine.h" 29#include "lsan/lsan_common.h" 30 31namespace __asan { 32 33struct AsanMapUnmapCallback { 34 void OnMap(uptr p, uptr size) const { 35 PoisonShadow(p, size, kAsanHeapLeftRedzoneMagic); 36 // Statistics. 37 AsanStats &thread_stats = GetCurrentThreadStats(); 38 thread_stats.mmaps++; 39 thread_stats.mmaped += size; 40 } 41 void OnUnmap(uptr p, uptr size) const { 42 PoisonShadow(p, size, 0); 43 // We are about to unmap a chunk of user memory. 44 // Mark the corresponding shadow memory as not needed. 45 // Since asan's mapping is compacting, the shadow chunk may be 46 // not page-aligned, so we only flush the page-aligned portion. 47 uptr page_size = GetPageSizeCached(); 48 uptr shadow_beg = RoundUpTo(MemToShadow(p), page_size); 49 uptr shadow_end = RoundDownTo(MemToShadow(p + size), page_size); 50 FlushUnneededShadowMemory(shadow_beg, shadow_end - shadow_beg); 51 // Statistics. 52 AsanStats &thread_stats = GetCurrentThreadStats(); 53 thread_stats.munmaps++; 54 thread_stats.munmaped += size; 55 } 56}; 57 58#if SANITIZER_WORDSIZE == 64 59#if defined(__powerpc64__) 60const uptr kAllocatorSpace = 0xa0000000000ULL; 61const uptr kAllocatorSize = 0x20000000000ULL; // 2T. 62#else 63const uptr kAllocatorSpace = 0x600000000000ULL; 64const uptr kAllocatorSize = 0x40000000000ULL; // 4T. 65#endif 66typedef DefaultSizeClassMap SizeClassMap; 67typedef SizeClassAllocator64<kAllocatorSpace, kAllocatorSize, 0 /*metadata*/, 68 SizeClassMap, AsanMapUnmapCallback> PrimaryAllocator; 69#elif SANITIZER_WORDSIZE == 32 70static const u64 kAddressSpaceSize = 1ULL << 32; 71typedef CompactSizeClassMap SizeClassMap; 72static const uptr kRegionSizeLog = 20; 73static const uptr kFlatByteMapSize = kAddressSpaceSize >> kRegionSizeLog; 74typedef SizeClassAllocator32<0, kAddressSpaceSize, 16, 75 SizeClassMap, kRegionSizeLog, 76 FlatByteMap<kFlatByteMapSize>, 77 AsanMapUnmapCallback> PrimaryAllocator; 78#endif 79 80typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; 81typedef LargeMmapAllocator<AsanMapUnmapCallback> SecondaryAllocator; 82typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, 83 SecondaryAllocator> Allocator; 84 85// We can not use THREADLOCAL because it is not supported on some of the 86// platforms we care about (OSX 10.6, Android). 87// static THREADLOCAL AllocatorCache cache; 88AllocatorCache *GetAllocatorCache(AsanThreadLocalMallocStorage *ms) { 89 CHECK(ms); 90 CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator2_cache)); 91 return reinterpret_cast<AllocatorCache *>(ms->allocator2_cache); 92} 93 94static Allocator allocator; 95 96static const uptr kMaxAllowedMallocSize = 97 FIRST_32_SECOND_64(3UL << 30, 8UL << 30); 98 99static const uptr kMaxThreadLocalQuarantine = 100 FIRST_32_SECOND_64(1 << 18, 1 << 20); 101 102// Every chunk of memory allocated by this allocator can be in one of 3 states: 103// CHUNK_AVAILABLE: the chunk is in the free list and ready to be allocated. 104// CHUNK_ALLOCATED: the chunk is allocated and not yet freed. 105// CHUNK_QUARANTINE: the chunk was freed and put into quarantine zone. 106enum { 107 CHUNK_AVAILABLE = 0, // 0 is the default value even if we didn't set it. 108 CHUNK_ALLOCATED = 2, 109 CHUNK_QUARANTINE = 3 110}; 111 112// Valid redzone sizes are 16, 32, 64, ... 2048, so we encode them in 3 bits. 113// We use adaptive redzones: for larger allocation larger redzones are used. 114static u32 RZLog2Size(u32 rz_log) { 115 CHECK_LT(rz_log, 8); 116 return 16 << rz_log; 117} 118 119static u32 RZSize2Log(u32 rz_size) { 120 CHECK_GE(rz_size, 16); 121 CHECK_LE(rz_size, 2048); 122 CHECK(IsPowerOfTwo(rz_size)); 123 u32 res = Log2(rz_size) - 4; 124 CHECK_EQ(rz_size, RZLog2Size(res)); 125 return res; 126} 127 128static uptr ComputeRZLog(uptr user_requested_size) { 129 u32 rz_log = 130 user_requested_size <= 64 - 16 ? 0 : 131 user_requested_size <= 128 - 32 ? 1 : 132 user_requested_size <= 512 - 64 ? 2 : 133 user_requested_size <= 4096 - 128 ? 3 : 134 user_requested_size <= (1 << 14) - 256 ? 4 : 135 user_requested_size <= (1 << 15) - 512 ? 5 : 136 user_requested_size <= (1 << 16) - 1024 ? 6 : 7; 137 return Max(rz_log, RZSize2Log(flags()->redzone)); 138} 139 140// The memory chunk allocated from the underlying allocator looks like this: 141// L L L L L L H H U U U U U U R R 142// L -- left redzone words (0 or more bytes) 143// H -- ChunkHeader (16 bytes), which is also a part of the left redzone. 144// U -- user memory. 145// R -- right redzone (0 or more bytes) 146// ChunkBase consists of ChunkHeader and other bytes that overlap with user 147// memory. 148 149// If a memory chunk is allocated by memalign and we had to increase the 150// allocation size to achieve the proper alignment, then we store this magic 151// value in the first uptr word of the memory block and store the address of 152// ChunkBase in the next uptr. 153// M B ? ? ? L L L L L L H H U U U U U U 154// M -- magic value kMemalignMagic 155// B -- address of ChunkHeader pointing to the first 'H' 156static const uptr kMemalignMagic = 0xCC6E96B9; 157 158struct ChunkHeader { 159 // 1-st 8 bytes. 160 u32 chunk_state : 8; // Must be first. 161 u32 alloc_tid : 24; 162 163 u32 free_tid : 24; 164 u32 from_memalign : 1; 165 u32 alloc_type : 2; 166 u32 rz_log : 3; 167 u32 lsan_tag : 2; 168 // 2-nd 8 bytes 169 // This field is used for small sizes. For large sizes it is equal to 170 // SizeClassMap::kMaxSize and the actual size is stored in the 171 // SecondaryAllocator's metadata. 172 u32 user_requested_size; 173 u32 alloc_context_id; 174}; 175 176struct ChunkBase : ChunkHeader { 177 // Header2, intersects with user memory. 178 u32 free_context_id; 179}; 180 181static const uptr kChunkHeaderSize = sizeof(ChunkHeader); 182static const uptr kChunkHeader2Size = sizeof(ChunkBase) - kChunkHeaderSize; 183COMPILER_CHECK(kChunkHeaderSize == 16); 184COMPILER_CHECK(kChunkHeader2Size <= 16); 185 186struct AsanChunk: ChunkBase { 187 uptr Beg() { return reinterpret_cast<uptr>(this) + kChunkHeaderSize; } 188 uptr UsedSize() { 189 if (user_requested_size != SizeClassMap::kMaxSize) 190 return user_requested_size; 191 return *reinterpret_cast<uptr *>(allocator.GetMetaData(AllocBeg())); 192 } 193 void *AllocBeg() { 194 if (from_memalign) 195 return allocator.GetBlockBegin(reinterpret_cast<void *>(this)); 196 return reinterpret_cast<void*>(Beg() - RZLog2Size(rz_log)); 197 } 198 // If we don't use stack depot, we store the alloc/free stack traces 199 // in the chunk itself. 200 u32 *AllocStackBeg() { 201 return (u32*)(Beg() - RZLog2Size(rz_log)); 202 } 203 uptr AllocStackSize() { 204 CHECK_LE(RZLog2Size(rz_log), kChunkHeaderSize); 205 return (RZLog2Size(rz_log) - kChunkHeaderSize) / sizeof(u32); 206 } 207 u32 *FreeStackBeg() { 208 return (u32*)(Beg() + kChunkHeader2Size); 209 } 210 uptr FreeStackSize() { 211 if (user_requested_size < kChunkHeader2Size) return 0; 212 uptr available = RoundUpTo(user_requested_size, SHADOW_GRANULARITY); 213 return (available - kChunkHeader2Size) / sizeof(u32); 214 } 215 bool AddrIsInside(uptr addr) { 216 return (addr >= Beg()) && (addr < Beg() + UsedSize()); 217 } 218}; 219 220uptr AsanChunkView::Beg() { return chunk_->Beg(); } 221uptr AsanChunkView::End() { return Beg() + UsedSize(); } 222uptr AsanChunkView::UsedSize() { return chunk_->UsedSize(); } 223uptr AsanChunkView::AllocTid() { return chunk_->alloc_tid; } 224uptr AsanChunkView::FreeTid() { return chunk_->free_tid; } 225 226static void GetStackTraceFromId(u32 id, StackTrace *stack) { 227 CHECK(id); 228 uptr size = 0; 229 const uptr *trace = StackDepotGet(id, &size); 230 CHECK_LT(size, kStackTraceMax); 231 internal_memcpy(stack->trace, trace, sizeof(uptr) * size); 232 stack->size = size; 233} 234 235void AsanChunkView::GetAllocStack(StackTrace *stack) { 236 if (flags()->use_stack_depot) 237 GetStackTraceFromId(chunk_->alloc_context_id, stack); 238 else 239 StackTrace::UncompressStack(stack, chunk_->AllocStackBeg(), 240 chunk_->AllocStackSize()); 241} 242 243void AsanChunkView::GetFreeStack(StackTrace *stack) { 244 if (flags()->use_stack_depot) 245 GetStackTraceFromId(chunk_->free_context_id, stack); 246 else 247 StackTrace::UncompressStack(stack, chunk_->FreeStackBeg(), 248 chunk_->FreeStackSize()); 249} 250 251struct QuarantineCallback; 252typedef Quarantine<QuarantineCallback, AsanChunk> AsanQuarantine; 253typedef AsanQuarantine::Cache QuarantineCache; 254static AsanQuarantine quarantine(LINKER_INITIALIZED); 255static QuarantineCache fallback_quarantine_cache(LINKER_INITIALIZED); 256static AllocatorCache fallback_allocator_cache; 257static SpinMutex fallback_mutex; 258 259QuarantineCache *GetQuarantineCache(AsanThreadLocalMallocStorage *ms) { 260 CHECK(ms); 261 CHECK_LE(sizeof(QuarantineCache), sizeof(ms->quarantine_cache)); 262 return reinterpret_cast<QuarantineCache *>(ms->quarantine_cache); 263} 264 265struct QuarantineCallback { 266 explicit QuarantineCallback(AllocatorCache *cache) 267 : cache_(cache) { 268 } 269 270 void Recycle(AsanChunk *m) { 271 CHECK_EQ(m->chunk_state, CHUNK_QUARANTINE); 272 atomic_store((atomic_uint8_t*)m, CHUNK_AVAILABLE, memory_order_relaxed); 273 CHECK_NE(m->alloc_tid, kInvalidTid); 274 CHECK_NE(m->free_tid, kInvalidTid); 275 PoisonShadow(m->Beg(), 276 RoundUpTo(m->UsedSize(), SHADOW_GRANULARITY), 277 kAsanHeapLeftRedzoneMagic); 278 void *p = reinterpret_cast<void *>(m->AllocBeg()); 279 if (m->from_memalign) { 280 uptr *memalign_magic = reinterpret_cast<uptr *>(p); 281 CHECK_EQ(memalign_magic[0], kMemalignMagic); 282 CHECK_EQ(memalign_magic[1], reinterpret_cast<uptr>(m)); 283 } 284 285 // Statistics. 286 AsanStats &thread_stats = GetCurrentThreadStats(); 287 thread_stats.real_frees++; 288 thread_stats.really_freed += m->UsedSize(); 289 290 allocator.Deallocate(cache_, p); 291 } 292 293 void *Allocate(uptr size) { 294 return allocator.Allocate(cache_, size, 1, false); 295 } 296 297 void Deallocate(void *p) { 298 allocator.Deallocate(cache_, p); 299 } 300 301 AllocatorCache *cache_; 302}; 303 304void InitializeAllocator() { 305 allocator.Init(); 306 quarantine.Init((uptr)flags()->quarantine_size, kMaxThreadLocalQuarantine); 307} 308 309static void *Allocate(uptr size, uptr alignment, StackTrace *stack, 310 AllocType alloc_type, bool can_fill) { 311 if (!asan_inited) 312 __asan_init(); 313 Flags &fl = *flags(); 314 CHECK(stack); 315 const uptr min_alignment = SHADOW_GRANULARITY; 316 if (alignment < min_alignment) 317 alignment = min_alignment; 318 if (size == 0) { 319 // We'd be happy to avoid allocating memory for zero-size requests, but 320 // some programs/tests depend on this behavior and assume that malloc would 321 // not return NULL even for zero-size allocations. Moreover, it looks like 322 // operator new should never return NULL, and results of consecutive "new" 323 // calls must be different even if the allocated size is zero. 324 size = 1; 325 } 326 CHECK(IsPowerOfTwo(alignment)); 327 uptr rz_log = ComputeRZLog(size); 328 uptr rz_size = RZLog2Size(rz_log); 329 uptr rounded_size = RoundUpTo(Max(size, kChunkHeader2Size), alignment); 330 uptr needed_size = rounded_size + rz_size; 331 if (alignment > min_alignment) 332 needed_size += alignment; 333 bool using_primary_allocator = true; 334 // If we are allocating from the secondary allocator, there will be no 335 // automatic right redzone, so add the right redzone manually. 336 if (!PrimaryAllocator::CanAllocate(needed_size, alignment)) { 337 needed_size += rz_size; 338 using_primary_allocator = false; 339 } 340 CHECK(IsAligned(needed_size, min_alignment)); 341 if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize) { 342 Report("WARNING: AddressSanitizer failed to allocate %p bytes\n", 343 (void*)size); 344 return 0; 345 } 346 347 AsanThread *t = GetCurrentThread(); 348 void *allocated; 349 if (t) { 350 AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage()); 351 allocated = allocator.Allocate(cache, needed_size, 8, false); 352 } else { 353 SpinMutexLock l(&fallback_mutex); 354 AllocatorCache *cache = &fallback_allocator_cache; 355 allocated = allocator.Allocate(cache, needed_size, 8, false); 356 } 357 uptr alloc_beg = reinterpret_cast<uptr>(allocated); 358 // Clear the first allocated word (an old kMemalignMagic may still be there). 359 reinterpret_cast<uptr *>(alloc_beg)[0] = 0; 360 uptr alloc_end = alloc_beg + needed_size; 361 uptr beg_plus_redzone = alloc_beg + rz_size; 362 uptr user_beg = beg_plus_redzone; 363 if (!IsAligned(user_beg, alignment)) 364 user_beg = RoundUpTo(user_beg, alignment); 365 uptr user_end = user_beg + size; 366 CHECK_LE(user_end, alloc_end); 367 uptr chunk_beg = user_beg - kChunkHeaderSize; 368 AsanChunk *m = reinterpret_cast<AsanChunk *>(chunk_beg); 369 m->alloc_type = alloc_type; 370 m->rz_log = rz_log; 371 u32 alloc_tid = t ? t->tid() : 0; 372 m->alloc_tid = alloc_tid; 373 CHECK_EQ(alloc_tid, m->alloc_tid); // Does alloc_tid fit into the bitfield? 374 m->free_tid = kInvalidTid; 375 m->from_memalign = user_beg != beg_plus_redzone; 376 if (m->from_memalign) { 377 CHECK_LE(beg_plus_redzone + 2 * sizeof(uptr), user_beg); 378 uptr *memalign_magic = reinterpret_cast<uptr *>(alloc_beg); 379 memalign_magic[0] = kMemalignMagic; 380 memalign_magic[1] = chunk_beg; 381 } 382 if (using_primary_allocator) { 383 CHECK(size); 384 m->user_requested_size = size; 385 CHECK(allocator.FromPrimary(allocated)); 386 } else { 387 CHECK(!allocator.FromPrimary(allocated)); 388 m->user_requested_size = SizeClassMap::kMaxSize; 389 uptr *meta = reinterpret_cast<uptr *>(allocator.GetMetaData(allocated)); 390 meta[0] = size; 391 meta[1] = chunk_beg; 392 } 393 394 if (fl.use_stack_depot) { 395 m->alloc_context_id = StackDepotPut(stack->trace, stack->size); 396 } else { 397 m->alloc_context_id = 0; 398 StackTrace::CompressStack(stack, m->AllocStackBeg(), m->AllocStackSize()); 399 } 400 401 uptr size_rounded_down_to_granularity = RoundDownTo(size, SHADOW_GRANULARITY); 402 // Unpoison the bulk of the memory region. 403 if (size_rounded_down_to_granularity) 404 PoisonShadow(user_beg, size_rounded_down_to_granularity, 0); 405 // Deal with the end of the region if size is not aligned to granularity. 406 if (size != size_rounded_down_to_granularity && fl.poison_heap) { 407 u8 *shadow = (u8*)MemToShadow(user_beg + size_rounded_down_to_granularity); 408 *shadow = size & (SHADOW_GRANULARITY - 1); 409 } 410 411 AsanStats &thread_stats = GetCurrentThreadStats(); 412 thread_stats.mallocs++; 413 thread_stats.malloced += size; 414 thread_stats.malloced_redzones += needed_size - size; 415 uptr class_id = Min(kNumberOfSizeClasses, SizeClassMap::ClassID(needed_size)); 416 thread_stats.malloced_by_size[class_id]++; 417 if (needed_size > SizeClassMap::kMaxSize) 418 thread_stats.malloc_large++; 419 420 void *res = reinterpret_cast<void *>(user_beg); 421 if (can_fill && fl.max_malloc_fill_size) { 422 uptr fill_size = Min(size, (uptr)fl.max_malloc_fill_size); 423 REAL(memset)(res, fl.malloc_fill_byte, fill_size); 424 } 425 m->lsan_tag = 426 t->lsan_disabled() ? __lsan::kSuppressed : __lsan::kDirectlyLeaked; 427 // Must be the last mutation of metadata in this function. 428 atomic_store((atomic_uint8_t *)m, CHUNK_ALLOCATED, memory_order_release); 429 ASAN_MALLOC_HOOK(res, size); 430 return res; 431} 432 433static void AtomicallySetQuarantineFlag(AsanChunk *m, 434 void *ptr, StackTrace *stack) { 435 u8 old_chunk_state = CHUNK_ALLOCATED; 436 // Flip the chunk_state atomically to avoid race on double-free. 437 if (!atomic_compare_exchange_strong((atomic_uint8_t*)m, &old_chunk_state, 438 CHUNK_QUARANTINE, memory_order_acquire)) { 439 if (old_chunk_state == CHUNK_QUARANTINE) 440 ReportDoubleFree((uptr)ptr, stack); 441 else 442 ReportFreeNotMalloced((uptr)ptr, stack); 443 } 444 CHECK_EQ(CHUNK_ALLOCATED, old_chunk_state); 445} 446 447// Expects the chunk to already be marked as quarantined by using 448// AtomicallySetQuarantineFlag. 449static void QuarantineChunk(AsanChunk *m, void *ptr, 450 StackTrace *stack, AllocType alloc_type) { 451 CHECK_EQ(m->chunk_state, CHUNK_QUARANTINE); 452 453 // FIXME: if the free hook produces an ASan report (e.g. due to a bug), 454 // printing the report may crash as the AsanChunk free-related fields have not 455 // been updated yet. We might need to introduce yet another chunk state to 456 // handle this correctly, but don't want to yet. 457 ASAN_FREE_HOOK(ptr); 458 459 if (m->alloc_type != alloc_type && flags()->alloc_dealloc_mismatch) 460 ReportAllocTypeMismatch((uptr)ptr, stack, 461 (AllocType)m->alloc_type, (AllocType)alloc_type); 462 463 CHECK_GE(m->alloc_tid, 0); 464 if (SANITIZER_WORDSIZE == 64) // On 32-bits this resides in user area. 465 CHECK_EQ(m->free_tid, kInvalidTid); 466 AsanThread *t = GetCurrentThread(); 467 m->free_tid = t ? t->tid() : 0; 468 if (flags()->use_stack_depot) { 469 m->free_context_id = StackDepotPut(stack->trace, stack->size); 470 } else { 471 m->free_context_id = 0; 472 StackTrace::CompressStack(stack, m->FreeStackBeg(), m->FreeStackSize()); 473 } 474 // Poison the region. 475 PoisonShadow(m->Beg(), 476 RoundUpTo(m->UsedSize(), SHADOW_GRANULARITY), 477 kAsanHeapFreeMagic); 478 479 AsanStats &thread_stats = GetCurrentThreadStats(); 480 thread_stats.frees++; 481 thread_stats.freed += m->UsedSize(); 482 483 // Push into quarantine. 484 if (t) { 485 AsanThreadLocalMallocStorage *ms = &t->malloc_storage(); 486 AllocatorCache *ac = GetAllocatorCache(ms); 487 quarantine.Put(GetQuarantineCache(ms), QuarantineCallback(ac), 488 m, m->UsedSize()); 489 } else { 490 SpinMutexLock l(&fallback_mutex); 491 AllocatorCache *ac = &fallback_allocator_cache; 492 quarantine.Put(&fallback_quarantine_cache, QuarantineCallback(ac), 493 m, m->UsedSize()); 494 } 495} 496 497static void Deallocate(void *ptr, StackTrace *stack, AllocType alloc_type) { 498 uptr p = reinterpret_cast<uptr>(ptr); 499 if (p == 0) return; 500 501 uptr chunk_beg = p - kChunkHeaderSize; 502 AsanChunk *m = reinterpret_cast<AsanChunk *>(chunk_beg); 503 // Must mark the chunk as quarantined before any changes to its metadata. 504 AtomicallySetQuarantineFlag(m, ptr, stack); 505 QuarantineChunk(m, ptr, stack, alloc_type); 506} 507 508static void *Reallocate(void *old_ptr, uptr new_size, StackTrace *stack) { 509 CHECK(old_ptr && new_size); 510 uptr p = reinterpret_cast<uptr>(old_ptr); 511 uptr chunk_beg = p - kChunkHeaderSize; 512 AsanChunk *m = reinterpret_cast<AsanChunk *>(chunk_beg); 513 514 AsanStats &thread_stats = GetCurrentThreadStats(); 515 thread_stats.reallocs++; 516 thread_stats.realloced += new_size; 517 518 // Must mark the chunk as quarantined before any changes to its metadata. 519 // This also ensures that other threads can't deallocate it in the meantime. 520 AtomicallySetQuarantineFlag(m, old_ptr, stack); 521 522 uptr old_size = m->UsedSize(); 523 uptr memcpy_size = Min(new_size, old_size); 524 void *new_ptr = Allocate(new_size, 8, stack, FROM_MALLOC, true); 525 if (new_ptr) { 526 CHECK_NE(REAL(memcpy), (void*)0); 527 REAL(memcpy)(new_ptr, old_ptr, memcpy_size); 528 QuarantineChunk(m, old_ptr, stack, FROM_MALLOC); 529 } 530 return new_ptr; 531} 532 533// Assumes alloc_beg == allocator.GetBlockBegin(alloc_beg). 534static AsanChunk *GetAsanChunk(void *alloc_beg) { 535 if (!alloc_beg) return 0; 536 uptr *memalign_magic = reinterpret_cast<uptr *>(alloc_beg); 537 if (memalign_magic[0] == kMemalignMagic) { 538 AsanChunk *m = reinterpret_cast<AsanChunk *>(memalign_magic[1]); 539 CHECK(m->from_memalign); 540 return m; 541 } 542 if (!allocator.FromPrimary(alloc_beg)) { 543 uptr *meta = reinterpret_cast<uptr *>(allocator.GetMetaData(alloc_beg)); 544 AsanChunk *m = reinterpret_cast<AsanChunk *>(meta[1]); 545 return m; 546 } 547 uptr actual_size = 548 allocator.GetActuallyAllocatedSize(alloc_beg); 549 CHECK_LE(actual_size, SizeClassMap::kMaxSize); 550 // We know the actually allocted size, but we don't know the redzone size. 551 // Just try all possible redzone sizes. 552 for (u32 rz_log = 0; rz_log < 8; rz_log++) { 553 u32 rz_size = RZLog2Size(rz_log); 554 uptr max_possible_size = actual_size - rz_size; 555 if (ComputeRZLog(max_possible_size) != rz_log) 556 continue; 557 return reinterpret_cast<AsanChunk *>( 558 reinterpret_cast<uptr>(alloc_beg) + rz_size - kChunkHeaderSize); 559 } 560 return 0; 561} 562 563static AsanChunk *GetAsanChunkByAddr(uptr p) { 564 void *alloc_beg = allocator.GetBlockBegin(reinterpret_cast<void *>(p)); 565 return GetAsanChunk(alloc_beg); 566} 567 568// Allocator must be locked when this function is called. 569static AsanChunk *GetAsanChunkByAddrFastLocked(uptr p) { 570 void *alloc_beg = 571 allocator.GetBlockBeginFastLocked(reinterpret_cast<void *>(p)); 572 return GetAsanChunk(alloc_beg); 573} 574 575static uptr AllocationSize(uptr p) { 576 AsanChunk *m = GetAsanChunkByAddr(p); 577 if (!m) return 0; 578 if (m->chunk_state != CHUNK_ALLOCATED) return 0; 579 if (m->Beg() != p) return 0; 580 return m->UsedSize(); 581} 582 583// We have an address between two chunks, and we want to report just one. 584AsanChunk *ChooseChunk(uptr addr, 585 AsanChunk *left_chunk, AsanChunk *right_chunk) { 586 // Prefer an allocated chunk over freed chunk and freed chunk 587 // over available chunk. 588 if (left_chunk->chunk_state != right_chunk->chunk_state) { 589 if (left_chunk->chunk_state == CHUNK_ALLOCATED) 590 return left_chunk; 591 if (right_chunk->chunk_state == CHUNK_ALLOCATED) 592 return right_chunk; 593 if (left_chunk->chunk_state == CHUNK_QUARANTINE) 594 return left_chunk; 595 if (right_chunk->chunk_state == CHUNK_QUARANTINE) 596 return right_chunk; 597 } 598 // Same chunk_state: choose based on offset. 599 sptr l_offset = 0, r_offset = 0; 600 CHECK(AsanChunkView(left_chunk).AddrIsAtRight(addr, 1, &l_offset)); 601 CHECK(AsanChunkView(right_chunk).AddrIsAtLeft(addr, 1, &r_offset)); 602 if (l_offset < r_offset) 603 return left_chunk; 604 return right_chunk; 605} 606 607AsanChunkView FindHeapChunkByAddress(uptr addr) { 608 AsanChunk *m1 = GetAsanChunkByAddr(addr); 609 if (!m1) return AsanChunkView(m1); 610 sptr offset = 0; 611 if (AsanChunkView(m1).AddrIsAtLeft(addr, 1, &offset)) { 612 // The address is in the chunk's left redzone, so maybe it is actually 613 // a right buffer overflow from the other chunk to the left. 614 // Search a bit to the left to see if there is another chunk. 615 AsanChunk *m2 = 0; 616 for (uptr l = 1; l < GetPageSizeCached(); l++) { 617 m2 = GetAsanChunkByAddr(addr - l); 618 if (m2 == m1) continue; // Still the same chunk. 619 break; 620 } 621 if (m2 && AsanChunkView(m2).AddrIsAtRight(addr, 1, &offset)) 622 m1 = ChooseChunk(addr, m2, m1); 623 } 624 return AsanChunkView(m1); 625} 626 627void AsanThreadLocalMallocStorage::CommitBack() { 628 AllocatorCache *ac = GetAllocatorCache(this); 629 quarantine.Drain(GetQuarantineCache(this), QuarantineCallback(ac)); 630 allocator.SwallowCache(GetAllocatorCache(this)); 631} 632 633void PrintInternalAllocatorStats() { 634 allocator.PrintStats(); 635} 636 637SANITIZER_INTERFACE_ATTRIBUTE 638void *asan_memalign(uptr alignment, uptr size, StackTrace *stack, 639 AllocType alloc_type) { 640 return Allocate(size, alignment, stack, alloc_type, true); 641} 642 643SANITIZER_INTERFACE_ATTRIBUTE 644void asan_free(void *ptr, StackTrace *stack, AllocType alloc_type) { 645 Deallocate(ptr, stack, alloc_type); 646} 647 648SANITIZER_INTERFACE_ATTRIBUTE 649void *asan_malloc(uptr size, StackTrace *stack) { 650 return Allocate(size, 8, stack, FROM_MALLOC, true); 651} 652 653void *asan_calloc(uptr nmemb, uptr size, StackTrace *stack) { 654 if (CallocShouldReturnNullDueToOverflow(size, nmemb)) return 0; 655 void *ptr = Allocate(nmemb * size, 8, stack, FROM_MALLOC, false); 656 // If the memory comes from the secondary allocator no need to clear it 657 // as it comes directly from mmap. 658 if (ptr && allocator.FromPrimary(ptr)) 659 REAL(memset)(ptr, 0, nmemb * size); 660 return ptr; 661} 662 663void *asan_realloc(void *p, uptr size, StackTrace *stack) { 664 if (p == 0) 665 return Allocate(size, 8, stack, FROM_MALLOC, true); 666 if (size == 0) { 667 Deallocate(p, stack, FROM_MALLOC); 668 return 0; 669 } 670 return Reallocate(p, size, stack); 671} 672 673void *asan_valloc(uptr size, StackTrace *stack) { 674 return Allocate(size, GetPageSizeCached(), stack, FROM_MALLOC, true); 675} 676 677void *asan_pvalloc(uptr size, StackTrace *stack) { 678 uptr PageSize = GetPageSizeCached(); 679 size = RoundUpTo(size, PageSize); 680 if (size == 0) { 681 // pvalloc(0) should allocate one page. 682 size = PageSize; 683 } 684 return Allocate(size, PageSize, stack, FROM_MALLOC, true); 685} 686 687int asan_posix_memalign(void **memptr, uptr alignment, uptr size, 688 StackTrace *stack) { 689 void *ptr = Allocate(size, alignment, stack, FROM_MALLOC, true); 690 CHECK(IsAligned((uptr)ptr, alignment)); 691 *memptr = ptr; 692 return 0; 693} 694 695uptr asan_malloc_usable_size(void *ptr, StackTrace *stack) { 696 CHECK(stack); 697 if (ptr == 0) return 0; 698 uptr usable_size = AllocationSize(reinterpret_cast<uptr>(ptr)); 699 if (flags()->check_malloc_usable_size && (usable_size == 0)) 700 ReportMallocUsableSizeNotOwned((uptr)ptr, stack); 701 return usable_size; 702} 703 704uptr asan_mz_size(const void *ptr) { 705 return AllocationSize(reinterpret_cast<uptr>(ptr)); 706} 707 708void asan_mz_force_lock() { 709 allocator.ForceLock(); 710 fallback_mutex.Lock(); 711} 712 713void asan_mz_force_unlock() { 714 fallback_mutex.Unlock(); 715 allocator.ForceUnlock(); 716} 717 718} // namespace __asan 719 720// --- Implementation of LSan-specific functions --- {{{1 721namespace __lsan { 722void LockAllocator() { 723 __asan::allocator.ForceLock(); 724} 725 726void UnlockAllocator() { 727 __asan::allocator.ForceUnlock(); 728} 729 730void GetAllocatorGlobalRange(uptr *begin, uptr *end) { 731 *begin = (uptr)&__asan::allocator; 732 *end = *begin + sizeof(__asan::allocator); 733} 734 735void *PointsIntoChunk(void* p) { 736 uptr addr = reinterpret_cast<uptr>(p); 737 __asan::AsanChunk *m = __asan::GetAsanChunkByAddrFastLocked(addr); 738 if (!m) return 0; 739 uptr chunk = m->Beg(); 740 if ((m->chunk_state == __asan::CHUNK_ALLOCATED) && m->AddrIsInside(addr)) 741 return reinterpret_cast<void *>(chunk); 742 return 0; 743} 744 745void *GetUserBegin(void *p) { 746 __asan::AsanChunk *m = 747 __asan::GetAsanChunkByAddrFastLocked(reinterpret_cast<uptr>(p)); 748 CHECK(m); 749 return reinterpret_cast<void *>(m->Beg()); 750} 751 752LsanMetadata::LsanMetadata(void *chunk) { 753 uptr addr = reinterpret_cast<uptr>(chunk); 754 metadata_ = reinterpret_cast<void *>(addr - __asan::kChunkHeaderSize); 755} 756 757bool LsanMetadata::allocated() const { 758 __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_); 759 return m->chunk_state == __asan::CHUNK_ALLOCATED; 760} 761 762ChunkTag LsanMetadata::tag() const { 763 __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_); 764 return static_cast<ChunkTag>(m->lsan_tag); 765} 766 767void LsanMetadata::set_tag(ChunkTag value) { 768 __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_); 769 m->lsan_tag = value; 770} 771 772uptr LsanMetadata::requested_size() const { 773 __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_); 774 return m->UsedSize(); 775} 776 777u32 LsanMetadata::stack_trace_id() const { 778 __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_); 779 return m->alloc_context_id; 780} 781 782template <typename Callable> void ForEachChunk(Callable const &callback) { 783 __asan::allocator.ForEachChunk(callback); 784} 785#if CAN_SANITIZE_LEAKS 786template void ForEachChunk<ProcessPlatformSpecificAllocationsCb>( 787 ProcessPlatformSpecificAllocationsCb const &callback); 788template void ForEachChunk<PrintLeakedCb>(PrintLeakedCb const &callback); 789template void ForEachChunk<CollectLeaksCb>(CollectLeaksCb const &callback); 790template void ForEachChunk<MarkIndirectlyLeakedCb>( 791 MarkIndirectlyLeakedCb const &callback); 792template void ForEachChunk<CollectSuppressedCb>( 793 CollectSuppressedCb const &callback); 794#endif // CAN_SANITIZE_LEAKS 795} // namespace __lsan 796 797extern "C" { 798void __lsan_disable() { 799 __asan_init(); 800 __asan::AsanThread *t = __asan::GetCurrentThread(); 801 CHECK(t); 802 t->disable_lsan(); 803} 804 805void __lsan_enable() { 806 __asan_init(); 807 __asan::AsanThread *t = __asan::GetCurrentThread(); 808 CHECK(t); 809 t->enable_lsan(); 810} 811} // extern "C" 812 813// ---------------------- Interface ---------------- {{{1 814using namespace __asan; // NOLINT 815 816// ASan allocator doesn't reserve extra bytes, so normally we would 817// just return "size". We don't want to expose our redzone sizes, etc here. 818uptr __asan_get_estimated_allocated_size(uptr size) { 819 return size; 820} 821 822bool __asan_get_ownership(const void *p) { 823 uptr ptr = reinterpret_cast<uptr>(p); 824 return (AllocationSize(ptr) > 0); 825} 826 827uptr __asan_get_allocated_size(const void *p) { 828 if (p == 0) return 0; 829 uptr ptr = reinterpret_cast<uptr>(p); 830 uptr allocated_size = AllocationSize(ptr); 831 // Die if p is not malloced or if it is already freed. 832 if (allocated_size == 0) { 833 GET_STACK_TRACE_FATAL_HERE; 834 ReportAsanGetAllocatedSizeNotOwned(ptr, &stack); 835 } 836 return allocated_size; 837} 838 839#if !SANITIZER_SUPPORTS_WEAK_HOOKS 840// Provide default (no-op) implementation of malloc hooks. 841extern "C" { 842SANITIZER_WEAK_ATTRIBUTE SANITIZER_INTERFACE_ATTRIBUTE 843void __asan_malloc_hook(void *ptr, uptr size) { 844 (void)ptr; 845 (void)size; 846} 847SANITIZER_WEAK_ATTRIBUTE SANITIZER_INTERFACE_ATTRIBUTE 848void __asan_free_hook(void *ptr) { 849 (void)ptr; 850} 851} // extern "C" 852#endif 853