msan_allocator.cc revision 5c48a8c4a8b3f7cd3f03f26c6aadc0ee606fc9b7
15f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===-- msan_allocator.cc --------------------------- ---------------------===// 25f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 35f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// The LLVM Compiler Infrastructure 45f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 50bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// This file is distributed under the University of Illinois Open Source 60bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// License. See LICENSE.TXT for details. 75f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 85f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 95f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// This file is a part of MemorySanitizer. 115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// MemorySanitizer allocator. 135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "sanitizer_common/sanitizer_allocator.h" 165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "sanitizer_common/sanitizer_stackdepot.h" 1794deaf675ae60e11c2d9475c6dbfd0c7123160f5Anders Carlsson#include "msan.h" 185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencernamespace __msan { 20161755a09898c95d21bfff33707da9ca41cd53c5John McCall 215baba9d98364a3525d6afa15a04cdad82fd6dd30John McCallstruct Metadata { 22709210feee317b8d6690dd1d15c2b74cfe55e261Ted Kremenek uptr requested_size; 23409c99edb8b623403fade6f3a9e9c86acda74455Anders Carlsson}; 245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 25525a05093a4816af961fe2bc6b8a81c17e2e26c2Chris Lattnerstatic const uptr kAllocatorSpace = 0x600000000000ULL; 263b8d116703db8018f855cbb4733ace426422623bNate Begemanstatic const uptr kAllocatorSize = 0x80000000000; // 8T. 27b648023da23e8b227cdda57a241db4c6f368726bDaniel Dunbarstatic const uptr kMetadataSize = sizeof(Metadata); 28c5ae899b4bbf65488445316c63168079177db0edSteve Naroff 295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencertypedef SizeClassAllocator64<kAllocatorSpace, kAllocatorSize, kMetadataSize, 305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer DefaultSizeClassMap> PrimaryAllocator; 31590b6646ef747d2f7b42e5f40487ff07642d7b6fChris Lattnertypedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; 32c44eec6dd29ee9415cbd38a35deff4c8b67abb6aAnders Carlssontypedef LargeMmapAllocator<> SecondaryAllocator; 33c4a1dea2dc56bd1357ec91b829a0b9e68229a13eDaniel Dunbartypedef CombinedAllocator<PrimaryAllocator, AllocatorCache, 34c4a1dea2dc56bd1357ec91b829a0b9e68229a13eDaniel Dunbar SecondaryAllocator> Allocator; 35c4a1dea2dc56bd1357ec91b829a0b9e68229a13eDaniel Dunbar 368e9bebdea69c590dedfbf27374114cb76fe12fbdDouglas Gregorstatic THREADLOCAL AllocatorCache cache; 37c4a1dea2dc56bd1357ec91b829a0b9e68229a13eDaniel Dunbarstatic Allocator allocator; 3856ee6896f2efebffb4a2cce5a7610cdf1eddbbbeSteve Naroff 39409c99edb8b623403fade6f3a9e9c86acda74455Anders Carlssonstatic int inited = 0; 4088a3514f36de96b19cdf50141c640df1a5f13f6cDouglas Gregor 4188a3514f36de96b19cdf50141c640df1a5f13f6cDouglas Gregorstatic inline void Init() { 42833ca991c1bfc967f0995974ca86f66ba1f666b5John McCall if (inited) return; 43d5532b6cfff2977e0c59fa6ead7f7973984a620dJohn McCall __msan_init(); 4488a3514f36de96b19cdf50141c640df1a5f13f6cDouglas Gregor inited = true; // this must happen before any threads are created. 457ab9d574d27ecee1f130e5755aa403e5ab529b6bAnders Carlsson allocator.Init(); 46f871d0cc377a1367b519a6cce26be74607566ebaJohn McCall} 477ab9d574d27ecee1f130e5755aa403e5ab529b6bAnders Carlsson 485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerstatic void *MsanAllocate(StackTrace *stack, uptr size, 495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer uptr alignment, bool zeroise) { 505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Init(); 515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer void *res = allocator.Allocate(&cache, size, alignment, false); 525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Metadata *meta = reinterpret_cast<Metadata*>(allocator.GetMetaData(res)); 535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer meta->requested_size = size; 54898574e7496ba8fd76290079d3a9d06954992734Douglas Gregor if (zeroise) 55bef0efd11bc4430a3ee437a3213cec5c18af855aChris Lattner __msan_clear_and_unpoison(res, size); 565baba9d98364a3525d6afa15a04cdad82fd6dd30John McCall else if (flags()->poison_in_malloc) 578e6285af719adc6f86d6faa235d22a08eb68ee3aJohn McCall __msan_poison(res, size); 588e6285af719adc6f86d6faa235d22a08eb68ee3aJohn McCall if (__msan_get_track_origins()) { 598e6285af719adc6f86d6faa235d22a08eb68ee3aJohn McCall u32 stack_id = StackDepotPut(stack->trace, stack->size); 608e6285af719adc6f86d6faa235d22a08eb68ee3aJohn McCall CHECK(stack_id); 618e6285af719adc6f86d6faa235d22a08eb68ee3aJohn McCall CHECK_EQ((stack_id >> 31), 0); // Higher bit is occupied by stack origins. 62898574e7496ba8fd76290079d3a9d06954992734Douglas Gregor __msan_set_origin(res, size, stack_id); 63898574e7496ba8fd76290079d3a9d06954992734Douglas Gregor } 64898574e7496ba8fd76290079d3a9d06954992734Douglas Gregor MSAN_MALLOC_HOOK(res, size); 650b7489194f9f89fac39d57211c1e7953ae50251fDouglas Gregor return res; 660b7489194f9f89fac39d57211c1e7953ae50251fDouglas Gregor} 670b7489194f9f89fac39d57211c1e7953ae50251fDouglas Gregor 688e6285af719adc6f86d6faa235d22a08eb68ee3aJohn McCallvoid MsanDeallocate(void *p) { 698e6285af719adc6f86d6faa235d22a08eb68ee3aJohn McCall CHECK(p); 708e6285af719adc6f86d6faa235d22a08eb68ee3aJohn McCall Init(); 718e6285af719adc6f86d6faa235d22a08eb68ee3aJohn McCall MSAN_FREE_HOOK(p); 728e6285af719adc6f86d6faa235d22a08eb68ee3aJohn McCall Metadata *meta = reinterpret_cast<Metadata*>(allocator.GetMetaData(p)); 738e6285af719adc6f86d6faa235d22a08eb68ee3aJohn McCall uptr size = meta->requested_size; 748e6285af719adc6f86d6faa235d22a08eb68ee3aJohn McCall meta->requested_size = 0; 758e6285af719adc6f86d6faa235d22a08eb68ee3aJohn McCall // This memory will not be reused by anyone else, so we are free to keep it 761eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // poisoned. 77f7c2aa0b049272d8f318988c1965760dcb852578Douglas Gregor __msan_poison(p, size); 781eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump if (__msan_get_track_origins()) 799d293dfc0ad7c44ae0b5eb9517f1ed8c8d8b7ff7Douglas Gregor __msan_set_origin(p, size, -1); 809d293dfc0ad7c44ae0b5eb9517f1ed8c8d8b7ff7Douglas Gregor allocator.Deallocate(&cache, p); 819d293dfc0ad7c44ae0b5eb9517f1ed8c8d8b7ff7Douglas Gregor} 829d293dfc0ad7c44ae0b5eb9517f1ed8c8d8b7ff7Douglas Gregor 839d293dfc0ad7c44ae0b5eb9517f1ed8c8d8b7ff7Douglas Gregorvoid *MsanReallocate(StackTrace *stack, void *old_p, uptr new_size, 849d293dfc0ad7c44ae0b5eb9517f1ed8c8d8b7ff7Douglas Gregor uptr alignment, bool zeroise) { 859d293dfc0ad7c44ae0b5eb9517f1ed8c8d8b7ff7Douglas Gregor if (!old_p) 86905d11d53aeb6b26744f44fedc2b2820c7a62df6Douglas Gregor return MsanAllocate(stack, new_size, alignment, zeroise); 878320aaaa01d931aa234fc3bce05b399ef41898d5Daniel Dunbar if (!new_size) { 88f7c2aa0b049272d8f318988c1965760dcb852578Douglas Gregor MsanDeallocate(old_p); 891eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump return 0; 909d293dfc0ad7c44ae0b5eb9517f1ed8c8d8b7ff7Douglas Gregor } 9177ed8e4edf6ed78c53fb20ec3210aff2a59c9d87Ted Kremenek Metadata *meta = reinterpret_cast<Metadata*>(allocator.GetMetaData(old_p)); 92898574e7496ba8fd76290079d3a9d06954992734Douglas Gregor uptr old_size = meta->requested_size; 93898574e7496ba8fd76290079d3a9d06954992734Douglas Gregor uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(old_p); 94898574e7496ba8fd76290079d3a9d06954992734Douglas Gregor if (new_size <= actually_allocated_size) { 951eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // We are not reallocating here. 96898574e7496ba8fd76290079d3a9d06954992734Douglas Gregor meta->requested_size = new_size; 97898574e7496ba8fd76290079d3a9d06954992734Douglas Gregor if (new_size > old_size) 98898574e7496ba8fd76290079d3a9d06954992734Douglas Gregor __msan_poison((char*)old_p + old_size, new_size - old_size); 998e6285af719adc6f86d6faa235d22a08eb68ee3aJohn McCall return old_p; 100898574e7496ba8fd76290079d3a9d06954992734Douglas Gregor } 1010b7489194f9f89fac39d57211c1e7953ae50251fDouglas Gregor uptr memcpy_size = Min(new_size, old_size); 1028e6285af719adc6f86d6faa235d22a08eb68ee3aJohn McCall void *new_p = MsanAllocate(stack, new_size, alignment, zeroise); 1030b7489194f9f89fac39d57211c1e7953ae50251fDouglas Gregor // Printf("realloc: old_size %zd new_size %zd\n", old_size, new_size); 104898574e7496ba8fd76290079d3a9d06954992734Douglas Gregor if (new_p) 105898574e7496ba8fd76290079d3a9d06954992734Douglas Gregor __msan_memcpy(new_p, old_p, memcpy_size); 106898574e7496ba8fd76290079d3a9d06954992734Douglas Gregor MsanDeallocate(old_p); 107898574e7496ba8fd76290079d3a9d06954992734Douglas Gregor return new_p; 108898574e7496ba8fd76290079d3a9d06954992734Douglas Gregor} 109898574e7496ba8fd76290079d3a9d06954992734Douglas Gregor 1101eb4433ac451dc16f4133a88af2d002ac26c58efMike Stumpstatic uptr AllocationSize(const void *p) { 111898574e7496ba8fd76290079d3a9d06954992734Douglas Gregor if (p == 0) 112898574e7496ba8fd76290079d3a9d06954992734Douglas Gregor return 0; 113898574e7496ba8fd76290079d3a9d06954992734Douglas Gregor const void *beg = allocator.GetBlockBegin(p); 114898574e7496ba8fd76290079d3a9d06954992734Douglas Gregor if (beg != p) 1158e6285af719adc6f86d6faa235d22a08eb68ee3aJohn McCall return 0; 116898574e7496ba8fd76290079d3a9d06954992734Douglas Gregor Metadata *b = (Metadata*)allocator.GetMetaData(p); 1170b7489194f9f89fac39d57211c1e7953ae50251fDouglas Gregor return b->requested_size; 1188e6285af719adc6f86d6faa235d22a08eb68ee3aJohn McCall} 1190b7489194f9f89fac39d57211c1e7953ae50251fDouglas Gregor 1205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} // namespace __msan 1215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerusing namespace __msan; 1235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spenceruptr __msan_get_current_allocated_bytes() { 1255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer u64 stats[AllocatorStatCount]; 1265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer allocator.GetStats(stats); 1275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer u64 m = stats[AllocatorStatMalloced]; 1281eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump u64 f = stats[AllocatorStatFreed]; 129026dc96ac6ece60da5e1b98e2a71bd0ff0939fd8Chris Lattner return m >= f ? m - f : 1; 130026dc96ac6ece60da5e1b98e2a71bd0ff0939fd8Chris Lattner} 131026dc96ac6ece60da5e1b98e2a71bd0ff0939fd8Chris Lattner 132026dc96ac6ece60da5e1b98e2a71bd0ff0939fd8Chris Lattneruptr __msan_get_heap_size() { 133026dc96ac6ece60da5e1b98e2a71bd0ff0939fd8Chris Lattner u64 stats[AllocatorStatCount]; 134df317bf71653eeb235da8337b1e8e790f9653aa4Mike Stump allocator.GetStats(stats); 1351eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump u64 m = stats[AllocatorStatMmapped]; 1365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer u64 f = stats[AllocatorStatUnmapped]; 1375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return m >= f ? m - f : 1; 1385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 1395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spenceruptr __msan_get_free_bytes() { 1415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return 1; 1425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 1435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spenceruptr __msan_get_unmapped_bytes() { 14508ad47cbd1f81fcb31dbc731c13b885a07e12704Bill Wendling return 1; 14676458501a8963fa11b91c9337a487de6871169b4Sebastian Redl} 1475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spenceruptr __msan_get_estimated_allocated_size(uptr size) { 1495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return size; 1505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 1515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 152fec0b49c3fa621fbf63e420f3d54a5bb3a0265d2Steve Naroffbool __msan_get_ownership(const void *p) { 15386f194083504938df72135b5b66bf0c5cafd9498Douglas Gregor return AllocationSize(p) != 0; 1542514a309204341798f96912ce7a90841bea59727Fariborz Jahanian} 155e9ff443040cb571ae2c5c2626c4dc9a9a812d84aFariborz Jahanian 156e873fb74219f48407ae0b8fa083aa7f0b6ff1427Douglas Gregoruptr __msan_get_allocated_size(const void *p) { 1575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return AllocationSize(p); 15828be73f74c9e241a23ea24fe5756623de6bf1084Chris Lattner} 15953202857c60214d80950a975e6e52aebf30bd16aEli Friedman