asan_allocator2.cc revision 6a11cc1bc665f13a0fcafe4a6a84761216675af7
12a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)//===-- asan_allocator2.cc ------------------------------------------------===// 25821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 35821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// The LLVM Compiler Infrastructure 45821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 55821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// This file is distributed under the University of Illinois Open Source 65821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// License. See LICENSE.TXT for details. 75821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 85821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)//===----------------------------------------------------------------------===// 95821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// This file is a part of AddressSanitizer, an address sanity checker. 115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Implementation of ASan's memory allocator, 2-nd version. 135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// This variant uses the allocator from sanitizer_common, i.e. the one shared 145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// with ThreadSanitizer and MemorySanitizer. 155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Status: under development, not enabled by default yet. 175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)//===----------------------------------------------------------------------===// 185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "asan_allocator.h" 195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if ASAN_ALLOCATOR_VERSION == 2 205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 212a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)#include "asan_mapping.h" 22bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch#include "asan_report.h" 232a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)#include "asan_thread.h" 24868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)#include "asan_thread_registry.h" 25868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)#include "sanitizer_common/sanitizer_allocator.h" 26868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)#include "sanitizer_common/sanitizer_internal_defs.h" 275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "sanitizer_common/sanitizer_list.h" 285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "sanitizer_common/sanitizer_stackdepot.h" 29eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#include "sanitizer_common/sanitizer_quarantine.h" 305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 312a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)namespace __asan { 325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)struct AsanMapUnmapCallback { 345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void OnMap(uptr p, uptr size) const { 355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PoisonShadow(p, size, kAsanHeapLeftRedzoneMagic); 365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Statistics. 375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AsanStats &thread_stats = asanThreadRegistry().GetCurrentThreadStats(); 387dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch thread_stats.mmaps++; 395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) thread_stats.mmaped += size; 405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void OnUnmap(uptr p, uptr size) const { 425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PoisonShadow(p, size, 0); 43c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) // We are about to unmap a chunk of user memory. 445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Mark the corresponding shadow memory as not needed. 455d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Since asan's mapping is compacting, the shadow chunk may be 46f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // not page-aligned, so we only flush the page-aligned portion. 472a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) uptr page_size = GetPageSizeCached(); 482a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) uptr shadow_beg = RoundUpTo(MemToShadow(p), page_size); 495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uptr shadow_end = RoundDownTo(MemToShadow(p + size), page_size); 505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FlushUnneededShadowMemory(shadow_beg, shadow_end - shadow_beg); 515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Statistics. 525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AsanStats &thread_stats = asanThreadRegistry().GetCurrentThreadStats(); 53116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch thread_stats.munmaps++; 54116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch thread_stats.munmaped += size; 555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}; 571320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if SANITIZER_WORDSIZE == 64 595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if defined(__powerpc64__) 605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)const uptr kAllocatorSpace = 0xa0000000000ULL; 615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#else 62c5cede9ae108bb15f6b7a8aea21c7e1fefa2834cBen Murdochconst uptr kAllocatorSpace = 0x600000000000ULL; 632a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)#endif 642a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)const uptr kAllocatorSize = 0x10000000000ULL; // 1T. 655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)typedef DefaultSizeClassMap SizeClassMap; 665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)typedef SizeClassAllocator64<kAllocatorSpace, kAllocatorSize, 0 /*metadata*/, 672a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) SizeClassMap, AsanMapUnmapCallback> PrimaryAllocator; 685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#elif SANITIZER_WORDSIZE == 32 695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static const u64 kAddressSpaceSize = 1ULL << 32; 705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)typedef CompactSizeClassMap SizeClassMap; 715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)typedef SizeClassAllocator32<0, kAddressSpaceSize, 16, 725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SizeClassMap, AsanMapUnmapCallback> PrimaryAllocator; 732a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)#endif 745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; 765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)typedef LargeMmapAllocator<AsanMapUnmapCallback> SecondaryAllocator; 775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, 785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SecondaryAllocator> Allocator; 795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// We can not use THREADLOCAL because it is not supported on some of the 815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// platforms we care about (OSX 10.6, Android). 825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// static THREADLOCAL AllocatorCache cache; 835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)AllocatorCache *GetAllocatorCache(AsanThreadLocalMallocStorage *ms) { 845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK(ms); 855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator2_cache)); 865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return reinterpret_cast<AllocatorCache *>(ms->allocator2_cache); 875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 8846d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) 892a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)static Allocator allocator; 90f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 91ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochstatic const uptr kMaxAllowedMallocSize = 925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FIRST_32_SECOND_64(3UL << 30, 8UL << 30); 93a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static const uptr kMaxThreadLocalQuarantine = 955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) FIRST_32_SECOND_64(1 << 18, 1 << 20); 965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 975d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)// Every chunk of memory allocated by this allocator can be in one of 3 states: 985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// CHUNK_AVAILABLE: the chunk is in the free list and ready to be allocated. 995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// CHUNK_ALLOCATED: the chunk is allocated and not yet freed. 1005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// CHUNK_QUARANTINE: the chunk was freed and put into quarantine zone. 1012a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)enum { 1025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHUNK_AVAILABLE = 0, // 0 is the default value even if we didn't set it. 1035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHUNK_ALLOCATED = 2, 1045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHUNK_QUARANTINE = 3 1055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}; 1065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Valid redzone sizes are 16, 32, 64, ... 2048, so we encode them in 3 bits. 1085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// We use adaptive redzones: for larger allocation larger redzones are used. 1095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static u32 RZLog2Size(u32 rz_log) { 1105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK_LT(rz_log, 8); 1115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 16 << rz_log; 112eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch} 1135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static u32 RZSize2Log(u32 rz_size) { 1155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK_GE(rz_size, 16); 1165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK_LE(rz_size, 2048); 1175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK(IsPowerOfTwo(rz_size)); 1185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) u32 res = __builtin_ctz(rz_size) - 4; 1195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK_EQ(rz_size, RZLog2Size(res)); 120116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch return res; 121d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)} 1225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static uptr ComputeRZLog(uptr user_requested_size) { 1241e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) u32 rz_log = 1252a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) user_requested_size <= 64 - 16 ? 0 : 1265f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) user_requested_size <= 128 - 32 ? 1 : 1275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) user_requested_size <= 512 - 64 ? 2 : 1285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) user_requested_size <= 4096 - 128 ? 3 : 1295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) user_requested_size <= (1 << 14) - 256 ? 4 : 1305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) user_requested_size <= (1 << 15) - 512 ? 5 : 1315d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) user_requested_size <= (1 << 16) - 1024 ? 6 : 7; 1325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return Max(rz_log, RZSize2Log(flags()->redzone)); 1335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 135c5cede9ae108bb15f6b7a8aea21c7e1fefa2834cBen Murdoch// The memory chunk allocated from the underlying allocator looks like this: 1365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// L L L L L L H H U U U U U U R R 1375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// L -- left redzone words (0 or more bytes) 1385d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)// H -- ChunkHeader (16 bytes), which is also a part of the left redzone. 1395d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)// U -- user memory. 1405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// R -- right redzone (0 or more bytes) 1415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// ChunkBase consists of ChunkHeader and other bytes that overlap with user 1425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// memory. 1432a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 1445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// If a memory chunk is allocated by memalign and we had to increase the 1455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// allocation size to achieve the proper alignment, then we store this magic 1465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// value in the first uptr word of the memory block and store the address of 1475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// ChunkBase in the next uptr. 1485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// M B ? ? ? L L L L L L H H U U U U U U 1495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// M -- magic value kMemalignMagic 1505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// B -- address of ChunkHeader pointing to the first 'H' 1512a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)static const uptr kMemalignMagic = 0xCC6E96B9; 152c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 1531320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tuccistruct ChunkHeader { 1541320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci // 1-st 8 bytes. 1551320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci u32 chunk_state : 8; // Must be first. 156cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) u32 alloc_tid : 24; 157cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 158f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) u32 free_tid : 24; 1595f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) u32 from_memalign : 1; 16058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) u32 alloc_type : 2; 161116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch u32 rz_log : 3; 16290dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) // 2-nd 8 bytes 16303b57e008b61dfcb1fbad3aea950ae0e001748b0Torne (Richard Coles) // This field is used for small sizes. For large sizes it is equal to 1645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // SizeClassMap::kMaxSize and the actual size is stored in the 1655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // SecondaryAllocator's metadata. 1665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) u32 user_requested_size; 1675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) u32 alloc_context_id; 1685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}; 1695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)struct ChunkBase : ChunkHeader { 1715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Header2, intersects with user memory. 1725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AsanChunk *next; 1735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) u32 free_context_id; 1745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}; 1755d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static const uptr kChunkHeaderSize = sizeof(ChunkHeader); 1775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static const uptr kChunkHeader2Size = sizeof(ChunkBase) - kChunkHeaderSize; 1785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)COMPILER_CHECK(kChunkHeaderSize == 16); 1795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)COMPILER_CHECK(kChunkHeader2Size <= 16); 1805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)struct AsanChunk: ChunkBase { 182ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch uptr Beg() { return reinterpret_cast<uptr>(this) + kChunkHeaderSize; } 1835d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) uptr UsedSize() { 1845d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if (user_requested_size != SizeClassMap::kMaxSize) 1852a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return user_requested_size; 186f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) return *reinterpret_cast<uptr *>(allocator.GetMetaData(AllocBeg())); 187f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) } 188c5cede9ae108bb15f6b7a8aea21c7e1fefa2834cBen Murdoch void *AllocBeg() { 1895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (from_memalign) 1905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return allocator.GetBlockBegin(reinterpret_cast<void *>(this)); 1915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return reinterpret_cast<void*>(Beg() - RZLog2Size(rz_log)); 192a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch } 1935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // We store the alloc/free stack traces in the chunk itself. 1942a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) u32 *AllocStackBeg() { 1955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (u32*)(Beg() - RZLog2Size(rz_log)); 1962a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) } 1975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uptr AllocStackSize() { 1985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK_LE(RZLog2Size(rz_log), kChunkHeaderSize); 1995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (RZLog2Size(rz_log) - kChunkHeaderSize) / sizeof(u32); 2005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) u32 *FreeStackBeg() { 2025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (u32*)(Beg() + kChunkHeader2Size); 203eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch } 2045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uptr FreeStackSize() { 2055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (user_requested_size < kChunkHeader2Size) return 0; 2065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uptr available = RoundUpTo(user_requested_size, SHADOW_GRANULARITY); 2075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (available - kChunkHeader2Size) / sizeof(u32); 2081320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci } 2095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}; 2105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)uptr AsanChunkView::Beg() { return chunk_->Beg(); } 2125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)uptr AsanChunkView::End() { return Beg() + UsedSize(); } 2135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)uptr AsanChunkView::UsedSize() { return chunk_->UsedSize(); } 2145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)uptr AsanChunkView::AllocTid() { return chunk_->alloc_tid; } 2155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)uptr AsanChunkView::FreeTid() { return chunk_->free_tid; } 2165d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void GetStackTraceFromId(u32 id, StackTrace *stack) { 2185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK(id); 2195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uptr size = 0; 2205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const uptr *trace = StackDepotGet(id, &size); 2215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK_LT(size, kStackTraceMax); 2225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) internal_memcpy(stack->trace, trace, sizeof(uptr) * size); 2235d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) stack->size = size; 2245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void AsanChunkView::GetAllocStack(StackTrace *stack) { 2275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (flags()->use_stack_depot) 22890dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) GetStackTraceFromId(chunk_->alloc_context_id, stack); 229f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) else 2305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) StackTrace::UncompressStack(stack, chunk_->AllocStackBeg(), 2315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) chunk_->AllocStackSize()); 2325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void AsanChunkView::GetFreeStack(StackTrace *stack) { 2355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (flags()->use_stack_depot) 2365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) GetStackTraceFromId(chunk_->free_context_id, stack); 2375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else 2385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) StackTrace::UncompressStack(stack, chunk_->FreeStackBeg(), 2395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) chunk_->FreeStackSize()); 2405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 242eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochstruct QuarantineCallback; 243eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochtypedef Quarantine<QuarantineCallback, AsanChunk> AsanQuarantine; 244eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochtypedef AsanQuarantine::Cache QuarantineCache; 245eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochstatic AsanQuarantine quarantine(LINKER_INITIALIZED); 246eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochstatic QuarantineCache fallback_quarantine_cache(LINKER_INITIALIZED); 247eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochstatic AllocatorCache fallback_allocator_cache; 2485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static SpinMutex fallback_mutex; 2495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)QuarantineCache *GetQuarantineCache(AsanThreadLocalMallocStorage *ms) { 2515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK(ms); 2525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK_LE(sizeof(QuarantineCache), sizeof(ms->quarantine_cache)); 2532a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return reinterpret_cast<QuarantineCache *>(ms->quarantine_cache); 2542a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)} 2555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)struct QuarantineCallback { 2572a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) explicit QuarantineCallback(AllocatorCache *cache) 258a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch : cache_(cache) { 2595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void Recycle(AsanChunk *m) { 2625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK(m->chunk_state == CHUNK_QUARANTINE); 2635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) m->chunk_state = CHUNK_AVAILABLE; 2645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK_NE(m->alloc_tid, kInvalidTid); 2655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK_NE(m->free_tid, kInvalidTid); 2665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PoisonShadow(m->Beg(), 2675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) RoundUpTo(m->UsedSize(), SHADOW_GRANULARITY), 2685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) kAsanHeapLeftRedzoneMagic); 2695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void *p = reinterpret_cast<void *>(m->AllocBeg()); 270a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch if (m->from_memalign) { 2715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uptr *memalign_magic = reinterpret_cast<uptr *>(p); 2725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK_EQ(memalign_magic[0], kMemalignMagic); 2735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK_EQ(memalign_magic[1], reinterpret_cast<uptr>(m)); 2745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Statistics. 2775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AsanStats &thread_stats = asanThreadRegistry().GetCurrentThreadStats(); 2785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) thread_stats.real_frees++; 279a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch thread_stats.really_freed += m->UsedSize(); 2805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2812a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) allocator.Deallocate(cache_, p); 2822a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) } 2835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void *Allocate(uptr size) { 285a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch return allocator.Allocate(cache_, size, 1, false); 2865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 287a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch 2885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void Deallocate(void *p) { 2895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) allocator.Deallocate(cache_, p); 2905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AllocatorCache *cache_; 2935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}; 2945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2952a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)void InitializeAllocator() { 2962a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) allocator.Init(); 2972a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) quarantine.Init((uptr)flags()->quarantine_size, kMaxThreadLocalQuarantine); 2985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 299a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch 3005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void *Allocate(uptr size, uptr alignment, StackTrace *stack, 3015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AllocType alloc_type) { 3025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (!asan_inited) 3032a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) __asan_init(); 3042a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) CHECK(stack); 3052a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) const uptr min_alignment = SHADOW_GRANULARITY; 3062a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if (alignment < min_alignment) 3072a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) alignment = min_alignment; 3082a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if (size == 0) { 3092a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) // We'd be happy to avoid allocating memory for zero-size requests, but 3102a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) // some programs/tests depend on this behavior and assume that malloc would 3112a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) // not return NULL even for zero-size allocations. Moreover, it looks like 3122a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) // operator new should never return NULL, and results of consecutive "new" 3132a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) // calls must be different even if the allocated size is zero. 3142a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) size = 1; 3152a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) } 3162a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) CHECK(IsPowerOfTwo(alignment)); 3172a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) uptr rz_log = ComputeRZLog(size); 3182a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) uptr rz_size = RZLog2Size(rz_log); 3192a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) uptr rounded_size = RoundUpTo(size, alignment); 3202a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if (rounded_size < kChunkHeader2Size) 3212a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) rounded_size = kChunkHeader2Size; 3222a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) uptr needed_size = rounded_size + rz_size; 3232a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if (alignment > min_alignment) 3242a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) needed_size += alignment; 3252a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) bool using_primary_allocator = true; 3262a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) // If we are allocating from the secondary allocator, there will be no 3275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // automatic right redzone, so add the right redzone manually. 3285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (!PrimaryAllocator::CanAllocate(needed_size, alignment)) { 3295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) needed_size += rz_size; 3305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) using_primary_allocator = false; 3315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 3325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK(IsAligned(needed_size, min_alignment)); 3335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize) { 334c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) Report("WARNING: AddressSanitizer failed to allocate %p bytes\n", 3355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (void*)size); 3365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 0; 3375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 338a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch 3395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AsanThread *t = asanThreadRegistry().GetCurrent(); 3405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void *allocated; 3415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (t) { 3425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage()); 343f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) allocated = allocator.Allocate(cache, needed_size, 8, false); 344f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) } else { 345c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) SpinMutexLock l(&fallback_mutex); 346c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) AllocatorCache *cache = &fallback_allocator_cache; 347c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) allocated = allocator.Allocate(cache, needed_size, 8, false); 348c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) } 349c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) uptr alloc_beg = reinterpret_cast<uptr>(allocated); 3505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Clear the first allocated word (an old kMemalignMagic may still be there). 351cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) reinterpret_cast<uptr *>(alloc_beg)[0] = 0; 3524e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) uptr alloc_end = alloc_beg + needed_size; 3534e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) uptr beg_plus_redzone = alloc_beg + rz_size; 3545d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) uptr user_beg = beg_plus_redzone; 3551320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci if (!IsAligned(user_beg, alignment)) 3561320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci user_beg = RoundUpTo(user_beg, alignment); 357868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) uptr user_end = user_beg + size; 358868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) CHECK_LE(user_end, alloc_end); 359868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) uptr chunk_beg = user_beg - kChunkHeaderSize; 360868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) AsanChunk *m = reinterpret_cast<AsanChunk *>(chunk_beg); 3615d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) m->chunk_state = CHUNK_ALLOCATED; 3625d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) m->alloc_type = alloc_type; 363868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) m->rz_log = rz_log; 364eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch u32 alloc_tid = t ? t->tid() : 0; 365eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch m->alloc_tid = alloc_tid; 366eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch CHECK_EQ(alloc_tid, m->alloc_tid); // Does alloc_tid fit into the bitfield? 367eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch m->free_tid = kInvalidTid; 368eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch m->from_memalign = user_beg != beg_plus_redzone; 369eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch if (m->from_memalign) { 3705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK_LE(beg_plus_redzone + 2 * sizeof(uptr), user_beg); 3715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uptr *memalign_magic = reinterpret_cast<uptr *>(alloc_beg); 3725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memalign_magic[0] = kMemalignMagic; 373c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) memalign_magic[1] = chunk_beg; 3743551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) } 3755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (using_primary_allocator) { 3760529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch CHECK(size); 3775f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) m->user_requested_size = size; 3785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK(allocator.FromPrimary(allocated)); 3795f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) } else { 3805f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) CHECK(!allocator.FromPrimary(allocated)); 3815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) m->user_requested_size = SizeClassMap::kMaxSize; 382cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) uptr *meta = reinterpret_cast<uptr *>(allocator.GetMetaData(allocated)); 3835f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) meta[0] = size; 3845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) meta[1] = chunk_beg; 3855f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) } 3865f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 3875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (flags()->use_stack_depot) { 3885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) m->alloc_context_id = StackDepotPut(stack->trace, stack->size); 3895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } else { 3905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) m->alloc_context_id = 0; 3915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) StackTrace::CompressStack(stack, m->AllocStackBeg(), m->AllocStackSize()); 3925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 3935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uptr size_rounded_down_to_granularity = RoundDownTo(size, SHADOW_GRANULARITY); 3955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Unpoison the bulk of the memory region. 3965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (size_rounded_down_to_granularity) 3975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PoisonShadow(user_beg, size_rounded_down_to_granularity, 0); 3982a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) // Deal with the end of the region if size is not aligned to granularity. 3992a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if (size != size_rounded_down_to_granularity && flags()->poison_heap) { 4002a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) u8 *shadow = (u8*)MemToShadow(user_beg + size_rounded_down_to_granularity); 4012a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) *shadow = size & (SHADOW_GRANULARITY - 1); 4021320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci } 4032a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 4042a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) AsanStats &thread_stats = asanThreadRegistry().GetCurrentThreadStats(); 4055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) thread_stats.mallocs++; 4065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) thread_stats.malloced += size; 4075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) thread_stats.malloced_redzones += needed_size - size; 4085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uptr class_id = Min(kNumberOfSizeClasses, SizeClassMap::ClassID(needed_size)); 4095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) thread_stats.malloced_by_size[class_id]++; 4105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (needed_size > SizeClassMap::kMaxSize) 4112a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) thread_stats.malloc_large++; 4125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 413a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) void *res = reinterpret_cast<void *>(user_beg); 414c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) ASAN_MALLOC_HOOK(res, size); 4155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return res; 4165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 4175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 418c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)static void Deallocate(void *ptr, StackTrace *stack, AllocType alloc_type) { 4195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uptr p = reinterpret_cast<uptr>(ptr); 4205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (p == 0) return; 4215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ASAN_FREE_HOOK(ptr); 4225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uptr chunk_beg = p - kChunkHeaderSize; 4235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AsanChunk *m = reinterpret_cast<AsanChunk *>(chunk_beg); 4245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Flip the chunk_state atomically to avoid race on double-free. 426f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) u8 old_chunk_state = atomic_exchange((atomic_uint8_t*)m, CHUNK_QUARANTINE, 427f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) memory_order_relaxed); 428f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 429f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) if (old_chunk_state == CHUNK_QUARANTINE) 430f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) ReportDoubleFree((uptr)ptr, stack); 4315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else if (old_chunk_state != CHUNK_ALLOCATED) 4325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ReportFreeNotMalloced((uptr)ptr, stack); 4335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK(old_chunk_state == CHUNK_ALLOCATED); 4345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (m->alloc_type != alloc_type && flags()->alloc_dealloc_mismatch) 4355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ReportAllocTypeMismatch((uptr)ptr, stack, 4365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (AllocType)m->alloc_type, (AllocType)alloc_type); 4375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK_GE(m->alloc_tid, 0); 4392a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if (SANITIZER_WORDSIZE == 64) // On 32-bits this resides in user area. 4405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK_EQ(m->free_tid, kInvalidTid); 4412a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) AsanThread *t = asanThreadRegistry().GetCurrent(); 442c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) m->free_tid = t ? t->tid() : 0; 443c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) if (flags()->use_stack_depot) { 4445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) m->free_context_id = StackDepotPut(stack->trace, stack->size); 4455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } else { 4465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) m->free_context_id = 0; 4475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) StackTrace::CompressStack(stack, m->FreeStackBeg(), m->FreeStackSize()); 448116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch } 449116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch CHECK(m->chunk_state == CHUNK_QUARANTINE); 450116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch // Poison the region. 451116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch PoisonShadow(m->Beg(), 452116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch RoundUpTo(m->UsedSize(), SHADOW_GRANULARITY), 453116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch kAsanHeapFreeMagic); 4545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AsanStats &thread_stats = asanThreadRegistry().GetCurrentThreadStats(); 4565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) thread_stats.frees++; 457cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) thread_stats.freed += m->UsedSize(); 458cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 459cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) // Push into quarantine. 460cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if (t) { 461cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) AsanThreadLocalMallocStorage *ms = &t->malloc_storage(); 4625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AllocatorCache *ac = GetAllocatorCache(ms); 463effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch quarantine.Put(GetQuarantineCache(ms), QuarantineCallback(ac), 4645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) m, m->UsedSize()); 4655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } else { 4662a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) SpinMutexLock l(&fallback_mutex); 4672a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) AllocatorCache *ac = &fallback_allocator_cache; 4682a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) quarantine.Put(&fallback_quarantine_cache, QuarantineCallback(ac), 4692a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) m, m->UsedSize()); 4705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 4725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void *Reallocate(void *old_ptr, uptr new_size, StackTrace *stack) { 4745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK(old_ptr && new_size); 4755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uptr p = reinterpret_cast<uptr>(old_ptr); 4765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uptr chunk_beg = p - kChunkHeaderSize; 4775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AsanChunk *m = reinterpret_cast<AsanChunk *>(chunk_beg); 4785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AsanStats &thread_stats = asanThreadRegistry().GetCurrentThreadStats(); 4805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) thread_stats.reallocs++; 4815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) thread_stats.realloced += new_size; 4825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4832a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) CHECK(m->chunk_state == CHUNK_ALLOCATED); 4845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uptr old_size = m->UsedSize(); 4855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uptr memcpy_size = Min(new_size, old_size); 4865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void *new_ptr = Allocate(new_size, 8, stack, FROM_MALLOC); 4875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (new_ptr) { 4885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK(REAL(memcpy) != 0); 4895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) REAL(memcpy)(new_ptr, old_ptr, memcpy_size); 4905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Deallocate(old_ptr, stack, FROM_MALLOC); 4915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return new_ptr; 4935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 4945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static AsanChunk *GetAsanChunkByAddr(uptr p) { 4965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void *ptr = reinterpret_cast<void *>(p); 4975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uptr alloc_beg = reinterpret_cast<uptr>(allocator.GetBlockBegin(ptr)); 4985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (!alloc_beg) return 0; 4995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uptr *memalign_magic = reinterpret_cast<uptr *>(alloc_beg); 5005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (memalign_magic[0] == kMemalignMagic) { 50190dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) AsanChunk *m = reinterpret_cast<AsanChunk *>(memalign_magic[1]); 50290dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) CHECK(m->from_memalign); 50390dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) return m; 50490dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) } 505eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch if (!allocator.FromPrimary(ptr)) { 506eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch uptr *meta = reinterpret_cast<uptr *>( 507eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch allocator.GetMetaData(reinterpret_cast<void *>(alloc_beg))); 508eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch AsanChunk *m = reinterpret_cast<AsanChunk *>(meta[1]); 5095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return m; 5105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 5115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uptr actual_size = allocator.GetActuallyAllocatedSize(ptr); 5125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK_LE(actual_size, SizeClassMap::kMaxSize); 5135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // We know the actually allocted size, but we don't know the redzone size. 5145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Just try all possible redzone sizes. 5155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for (u32 rz_log = 0; rz_log < 8; rz_log++) { 5165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) u32 rz_size = RZLog2Size(rz_log); 5175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uptr max_possible_size = actual_size - rz_size; 5185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (ComputeRZLog(max_possible_size) != rz_log) 5195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) continue; 520f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) return reinterpret_cast<AsanChunk *>( 521f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) alloc_beg + rz_size - kChunkHeaderSize); 522f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) } 523f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) return 0; 524f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)} 525f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 526f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)static uptr AllocationSize(uptr p) { 5275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AsanChunk *m = GetAsanChunkByAddr(p); 5285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (!m) return 0; 5295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (m->chunk_state != CHUNK_ALLOCATED) return 0; 5305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (m->Beg() != p) return 0; 5315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return m->UsedSize(); 5325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 5335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// We have an address between two chunks, and we want to report just one. 5355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)AsanChunk *ChooseChunk(uptr addr, 5365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AsanChunk *left_chunk, AsanChunk *right_chunk) { 5372a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) // Prefer an allocated chunk over freed chunk and freed chunk 5382a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) // over available chunk. 5395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (left_chunk->chunk_state != right_chunk->chunk_state) { 5405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (left_chunk->chunk_state == CHUNK_ALLOCATED) 5415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return left_chunk; 5425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (right_chunk->chunk_state == CHUNK_ALLOCATED) 5435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return right_chunk; 5445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (left_chunk->chunk_state == CHUNK_QUARANTINE) 5455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return left_chunk; 5465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (right_chunk->chunk_state == CHUNK_QUARANTINE) 5475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return right_chunk; 5485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 5495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Same chunk_state: choose based on offset. 5505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sptr l_offset = 0, r_offset = 0; 5515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK(AsanChunkView(left_chunk).AddrIsAtRight(addr, 1, &l_offset)); 5525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) CHECK(AsanChunkView(right_chunk).AddrIsAtLeft(addr, 1, &r_offset)); 5535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (l_offset < r_offset) 5545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return left_chunk; 5555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return right_chunk; 5565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 5575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)AsanChunkView FindHeapChunkByAddress(uptr addr) { 5595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AsanChunk *m1 = GetAsanChunkByAddr(addr); 5602a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if (!m1) return AsanChunkView(m1); 5615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sptr offset = 0; 5625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (AsanChunkView(m1).AddrIsAtLeft(addr, 1, &offset)) { 5635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // The address is in the chunk's left redzone, so maybe it is actually 5645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // a right buffer overflow from the other chunk to the left. 5655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Search a bit to the left to see if there is another chunk. 5665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AsanChunk *m2 = 0; 5675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for (uptr l = 1; l < GetPageSizeCached(); l++) { 568a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) m2 = GetAsanChunkByAddr(addr - l); 5692a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if (m2 == m1) continue; // Still the same chunk. 570a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) break; 5715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 5725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (m2 && AsanChunkView(m2).AddrIsAtRight(addr, 1, &offset)) 5735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) m1 = ChooseChunk(addr, m2, m1); 5745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 57503b57e008b61dfcb1fbad3aea950ae0e001748b0Torne (Richard Coles) return AsanChunkView(m1); 5761320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci} 57703b57e008b61dfcb1fbad3aea950ae0e001748b0Torne (Richard Coles) 57803b57e008b61dfcb1fbad3aea950ae0e001748b0Torne (Richard Coles)void AsanThreadLocalMallocStorage::CommitBack() { 5795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AllocatorCache *ac = GetAllocatorCache(this); 5805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) quarantine.Drain(GetQuarantineCache(this), QuarantineCallback(ac)); 5815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) allocator.SwallowCache(GetAllocatorCache(this)); 5825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 5835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void PrintInternalAllocatorStats() { 5852a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) allocator.PrintStats(); 5862a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)} 5875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)SANITIZER_INTERFACE_ATTRIBUTE 5892a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)void *asan_memalign(uptr alignment, uptr size, StackTrace *stack, 5902a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) AllocType alloc_type) { 5912a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return Allocate(size, alignment, stack, alloc_type); 5922a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)} 5932a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 5942a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)SANITIZER_INTERFACE_ATTRIBUTE 5952a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)void asan_free(void *ptr, StackTrace *stack, AllocType alloc_type) { 5965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Deallocate(ptr, stack, alloc_type); 5975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 5985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)SANITIZER_INTERFACE_ATTRIBUTE 6005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void *asan_malloc(uptr size, StackTrace *stack) { 6015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return Allocate(size, 8, stack, FROM_MALLOC); 6025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 603a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) 6045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void *asan_calloc(uptr nmemb, uptr size, StackTrace *stack) { 6055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (CallocShouldReturnNullDueToOverflow(size, nmemb)) return 0; 606a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) void *ptr = Allocate(nmemb * size, 8, stack, FROM_MALLOC); 607a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) if (ptr) 608a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) REAL(memset)(ptr, 0, nmemb * size); 6095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ptr; 6105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 6115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void *asan_realloc(void *p, uptr size, StackTrace *stack) { 6135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (p == 0) 6145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return Allocate(size, 8, stack, FROM_MALLOC); 6155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (size == 0) { 6165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Deallocate(p, stack, FROM_MALLOC); 6175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 0; 6185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 6195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return Reallocate(p, size, stack); 620eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch} 621eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 6225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void *asan_valloc(uptr size, StackTrace *stack) { 6235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return Allocate(size, GetPageSizeCached(), stack, FROM_MALLOC); 6245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 6253551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 6263551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)void *asan_pvalloc(uptr size, StackTrace *stack) { 6273551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) uptr PageSize = GetPageSizeCached(); 6283551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) size = RoundUpTo(size, PageSize); 6293551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) if (size == 0) { 6303551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // pvalloc(0) should allocate one page. 6313551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) size = PageSize; 6323551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) } 6333551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return Allocate(size, PageSize, stack, FROM_MALLOC); 6343551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)} 6353551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 6363551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)int asan_posix_memalign(void **memptr, uptr alignment, uptr size, 6373551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) StackTrace *stack) { 6383551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) void *ptr = Allocate(size, alignment, stack, FROM_MALLOC); 6393551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) CHECK(IsAligned((uptr)ptr, alignment)); 6403551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) *memptr = ptr; 6413551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return 0; 6423551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)} 643868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 644eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochuptr asan_malloc_usable_size(void *ptr, StackTrace *stack) { 645eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch CHECK(stack); 646868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) if (ptr == 0) return 0; 647868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) uptr usable_size = AllocationSize(reinterpret_cast<uptr>(ptr)); 6485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (flags()->check_malloc_usable_size && (usable_size == 0)) 649eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch ReportMallocUsableSizeNotOwned((uptr)ptr, stack); 650eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch return usable_size; 6515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 6525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)uptr asan_mz_size(const void *ptr) { 6545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return AllocationSize(reinterpret_cast<uptr>(ptr)); 6555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 6565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void asan_mz_force_lock() { 6585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) allocator.ForceLock(); 6595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fallback_mutex.Lock(); 6605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 6615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void asan_mz_force_unlock() { 6632a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) fallback_mutex.Unlock(); 6645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) allocator.ForceUnlock(); 6655d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)} 6665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} // namespace __asan 6685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// ---------------------- Interface ---------------- {{{1 6705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)using namespace __asan; // NOLINT 6715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// ASan allocator doesn't reserve extra bytes, so normally we would 6735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// just return "size". We don't want to expose our redzone sizes, etc here. 6745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)uptr __asan_get_estimated_allocated_size(uptr size) { 6755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return size; 6765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 6775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)bool __asan_get_ownership(const void *p) { 6790f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) uptr ptr = reinterpret_cast<uptr>(p); 6805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (AllocationSize(ptr) > 0); 6815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 6825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6832a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)uptr __asan_get_allocated_size(const void *p) { 6845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (p == 0) return 0; 6855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uptr ptr = reinterpret_cast<uptr>(p); 6865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uptr allocated_size = AllocationSize(ptr); 6875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Die if p is not malloced or if it is already freed. 6885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (allocated_size == 0) { 6895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) GET_STACK_TRACE_FATAL_HERE; 6905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ReportAsanGetAllocatedSizeNotOwned(ptr, &stack); 691eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch } 692eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch return allocated_size; 693eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch} 6945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if !SANITIZER_SUPPORTS_WEAK_HOOKS 6965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Provide default (no-op) implementation of malloc hooks. 6975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" { 6985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)SANITIZER_WEAK_ATTRIBUTE SANITIZER_INTERFACE_ATTRIBUTE 6995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void __asan_malloc_hook(void *ptr, uptr size) { 7005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (void)ptr; 7015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (void)size; 7025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 7035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)SANITIZER_WEAK_ATTRIBUTE SANITIZER_INTERFACE_ATTRIBUTE 7045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void __asan_free_hook(void *ptr) { 7055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (void)ptr; 7065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 7075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} // extern "C" 7085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 7095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif // ASAN_ALLOCATOR_VERSION 7125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)