tsan_mman.cc revision c58a43648cd6121c51a2e795a28e2ef90d7813e6
1603c4be006d8c53905d736bf1f19a49f5ce98276Alexey Samsonov//===-- tsan_mman.cc ------------------------------------------------------===// 27ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 37ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// The LLVM Compiler Infrastructure 47ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 57ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// This file is distributed under the University of Illinois Open Source 67ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// License. See LICENSE.TXT for details. 77ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 87ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany//===----------------------------------------------------------------------===// 97ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 107ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// This file is a part of ThreadSanitizer (TSan), a race detector. 117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany//===----------------------------------------------------------------------===// 136a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines#include "sanitizer_common/sanitizer_allocator_interface.h" 14f7667cc84cdd8923c0b6c7cfc92b7bd5692ce18cAlexey Samsonov#include "sanitizer_common/sanitizer_common.h" 152e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov#include "sanitizer_common/sanitizer_placement_new.h" 167ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_mman.h" 177ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_rtl.h" 187ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_report.h" 197ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_flags.h" 207ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 214f0ea398bdc99a4a32402057c23bbcc6d19a8eb4Alexey Samsonov// May be overriden by front-end. 22799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga NainarSANITIZER_WEAK_DEFAULT_IMPL 23799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainarvoid __sanitizer_malloc_hook(void *ptr, uptr size) { 246a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines (void)ptr; 256a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines (void)size; 266a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines} 274f0ea398bdc99a4a32402057c23bbcc6d19a8eb4Alexey Samsonov 28799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga NainarSANITIZER_WEAK_DEFAULT_IMPL 29799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainarvoid __sanitizer_free_hook(void *ptr) { 306a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines (void)ptr; 31f51c3860ce0a1ae81d0dc9da27db0693718db18eDmitry Vyukov} 32f51c3860ce0a1ae81d0dc9da27db0693718db18eDmitry Vyukov 336a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hinesnamespace __tsan { 34f51c3860ce0a1ae81d0dc9da27db0693718db18eDmitry Vyukov 35e93e5057023de89f1bad5de609efac39efc5da73Dmitry Vyukovstruct MapUnmapCallback { 36e93e5057023de89f1bad5de609efac39efc5da73Dmitry Vyukov void OnMap(uptr p, uptr size) const { } 37e93e5057023de89f1bad5de609efac39efc5da73Dmitry Vyukov void OnUnmap(uptr p, uptr size) const { 38e93e5057023de89f1bad5de609efac39efc5da73Dmitry Vyukov // We are about to unmap a chunk of user memory. 39e93e5057023de89f1bad5de609efac39efc5da73Dmitry Vyukov // Mark the corresponding shadow memory as not needed. 407ac33ac529ff93a57419f5ddf71b1fde68428577Dmitry Vyukov DontNeedShadowFor(p, size); 41799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar // Mark the corresponding meta shadow memory as not needed. 42799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar // Note the block does not contain any meta info at this point 43799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar // (this happens after free). 44799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar const uptr kMetaRatio = kMetaShadowCell / kMetaShadowSize; 45799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar const uptr kPageSize = GetPageSizeCached() * kMetaRatio; 46799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar // Block came from LargeMmapAllocator, so must be large. 47799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar // We rely on this in the calculations below. 48799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar CHECK_GE(size, 2 * kPageSize); 49799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar uptr diff = RoundUp(p, kPageSize) - p; 50799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar if (diff != 0) { 51799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar p += diff; 52799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar size -= diff; 53799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar } 54799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar diff = p + size - RoundDown(p + size, kPageSize); 55799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar if (diff != 0) 56799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar size -= diff; 57799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar FlushUnneededShadowMemory((uptr)MemToMeta(p), size / kMetaRatio); 58e93e5057023de89f1bad5de609efac39efc5da73Dmitry Vyukov } 59e93e5057023de89f1bad5de609efac39efc5da73Dmitry Vyukov}; 60e93e5057023de89f1bad5de609efac39efc5da73Dmitry Vyukov 61ff35f1d82b4f145b3477ef27a7a2e7b63c486988Dmitry Vyukovstatic char allocator_placeholder[sizeof(Allocator)] ALIGNED(64); 62ff35f1d82b4f145b3477ef27a7a2e7b63c486988Dmitry VyukovAllocator *allocator() { 632e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov return reinterpret_cast<Allocator*>(&allocator_placeholder); 642e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov} 652e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov 66c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainarstruct GlobalProc { 67c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Mutex mtx; 68c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar Processor *proc; 69c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 70c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar GlobalProc() 71c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar : mtx(MutexTypeGlobalProc, StatMtxGlobalProc) 72c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar , proc(ProcCreate()) { 73c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar } 74c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar}; 75c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 76c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainarstatic char global_proc_placeholder[sizeof(GlobalProc)] ALIGNED(64); 77c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarGlobalProc *global_proc() { 78c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar return reinterpret_cast<GlobalProc*>(&global_proc_placeholder); 79c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar} 80c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 81c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarScopedGlobalProcessor::ScopedGlobalProcessor() { 82c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar GlobalProc *gp = global_proc(); 83c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar ThreadState *thr = cur_thread(); 84c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar if (thr->proc()) 85c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar return; 86c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar // If we don't have a proc, use the global one. 87c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar // There are currently only two known case where this path is triggered: 88c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar // __interceptor_free 89c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar // __nptl_deallocate_tsd 90c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar // start_thread 91c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar // clone 92c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar // and: 93c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar // ResetRange 94c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar // __interceptor_munmap 95c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar // __deallocate_stack 96c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar // start_thread 97c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar // clone 98c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar // Ideally, we destroy thread state (and unwire proc) when a thread actually 99c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar // exits (i.e. when we join/wait it). Then we would not need the global proc 100c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar gp->mtx.Lock(); 101c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar ProcWire(gp->proc, thr); 102c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar} 103c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 104c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga NainarScopedGlobalProcessor::~ScopedGlobalProcessor() { 105c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar GlobalProc *gp = global_proc(); 106c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar ThreadState *thr = cur_thread(); 107c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar if (thr->proc() != gp->proc) 108c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar return; 109c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar ProcUnwire(gp->proc, thr); 110c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar gp->mtx.Unlock(); 111c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar} 112c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 1132e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukovvoid InitializeAllocator() { 11486277eb844c4983c81de62d7c050e92fe7155788Stephen Hines allocator()->Init(common_flags()->allocator_may_return_null); 1152e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov} 1162e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov 117c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainarvoid InitializeAllocatorLate() { 118c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar new(global_proc()) GlobalProc(); 119c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar} 120c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar 121c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainarvoid AllocatorProcStart(Processor *proc) { 122c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar allocator()->InitCache(&proc->alloc_cache); 123c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar internal_allocator()->InitCache(&proc->internal_alloc_cache); 124bdd844cb41718c27ef727a99a236191bc29a3df8Dmitry Vyukov} 125bdd844cb41718c27ef727a99a236191bc29a3df8Dmitry Vyukov 126c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainarvoid AllocatorProcFinish(Processor *proc) { 127c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar allocator()->DestroyCache(&proc->alloc_cache); 128c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar internal_allocator()->DestroyCache(&proc->internal_alloc_cache); 129bdd844cb41718c27ef727a99a236191bc29a3df8Dmitry Vyukov} 130bdd844cb41718c27ef727a99a236191bc29a3df8Dmitry Vyukov 131bdd844cb41718c27ef727a99a236191bc29a3df8Dmitry Vyukovvoid AllocatorPrintStats() { 132bdd844cb41718c27ef727a99a236191bc29a3df8Dmitry Vyukov allocator()->PrintStats(); 1332e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov} 1342e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov 1357ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystatic void SignalUnsafeCall(ThreadState *thr, uptr pc) { 136799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar if (atomic_load_relaxed(&thr->in_signal_handler) == 0 || 1376d1862363c88c183b0ed7740fca876342cf0474bStephen Hines !flags()->report_signal_unsafe) 1387ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 1396d1862363c88c183b0ed7740fca876342cf0474bStephen Hines VarSizeStackTrace stack; 1406d1862363c88c183b0ed7740fca876342cf0474bStephen Hines ObtainCurrentStack(thr, pc, &stack); 141799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar if (IsFiredSuppression(ctx, ReportTypeSignalUnsafe, stack)) 142799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar return; 1432bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov ThreadRegistryLock l(ctx->thread_registry); 1447ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ScopedReport rep(ReportTypeSignalUnsafe); 145799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar rep.AddStack(stack, true); 146799172d60d32feb1acba1a6867f3a9c39a999e5cPirama Arumuga Nainar OutputReport(thr, rep); 1477ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 1487ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1496d1862363c88c183b0ed7740fca876342cf0474bStephen Hinesvoid *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align, bool signal) { 1507423c7821ff30043def78c6cbb257bd2f4d9eef6Dmitry Vyukov if ((sz >= (1ull << 40)) || (align >= (1ull << 40))) 15186277eb844c4983c81de62d7c050e92fe7155788Stephen Hines return allocator()->ReturnNullOrDie(); 152c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar void *p = allocator()->Allocate(&thr->proc()->alloc_cache, sz, align); 1532e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov if (p == 0) 154efd958213d70188ae6f79afd79fe2c84956d24ffDmitry Vyukov return 0; 1556a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines if (ctx && ctx->initialized) 1566a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines OnUserAlloc(thr, pc, (uptr)p, sz, true); 1576d1862363c88c183b0ed7740fca876342cf0474bStephen Hines if (signal) 1586d1862363c88c183b0ed7740fca876342cf0474bStephen Hines SignalUnsafeCall(thr, pc); 1597ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return p; 1607ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 1617ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 16286277eb844c4983c81de62d7c050e92fe7155788Stephen Hinesvoid *user_calloc(ThreadState *thr, uptr pc, uptr size, uptr n) { 16386277eb844c4983c81de62d7c050e92fe7155788Stephen Hines if (CallocShouldReturnNullDueToOverflow(size, n)) 16486277eb844c4983c81de62d7c050e92fe7155788Stephen Hines return allocator()->ReturnNullOrDie(); 16586277eb844c4983c81de62d7c050e92fe7155788Stephen Hines void *p = user_alloc(thr, pc, n * size); 16686277eb844c4983c81de62d7c050e92fe7155788Stephen Hines if (p) 16786277eb844c4983c81de62d7c050e92fe7155788Stephen Hines internal_memset(p, 0, n * size); 16886277eb844c4983c81de62d7c050e92fe7155788Stephen Hines return p; 16986277eb844c4983c81de62d7c050e92fe7155788Stephen Hines} 17086277eb844c4983c81de62d7c050e92fe7155788Stephen Hines 1716d1862363c88c183b0ed7740fca876342cf0474bStephen Hinesvoid user_free(ThreadState *thr, uptr pc, void *p, bool signal) { 172c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar ScopedGlobalProcessor sgp; 1736a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines if (ctx && ctx->initialized) 1746a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines OnUserFree(thr, pc, (uptr)p, true); 175c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar allocator()->Deallocate(&thr->proc()->alloc_cache, p); 1766d1862363c88c183b0ed7740fca876342cf0474bStephen Hines if (signal) 1776d1862363c88c183b0ed7740fca876342cf0474bStephen Hines SignalUnsafeCall(thr, pc); 1787ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 1797ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1806a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hinesvoid OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) { 1816a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines DPrintf("#%d: alloc(%zu) = %p\n", thr->tid, sz, p); 1826a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines ctx->metamap.AllocBlock(thr, pc, p, sz); 1836a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines if (write && thr->ignore_reads_and_writes == 0) 1846a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines MemoryRangeImitateWrite(thr, pc, (uptr)p, sz); 1856a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines else 1866a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines MemoryResetRange(thr, pc, (uptr)p, sz); 1876a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines} 1886a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines 1896a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hinesvoid OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write) { 1906a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines CHECK_NE(p, (void*)0); 191c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar uptr sz = ctx->metamap.FreeBlock(thr->proc(), p); 1926a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines DPrintf("#%d: free(%p, %zu)\n", thr->tid, p, sz); 1936a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines if (write && thr->ignore_reads_and_writes == 0) 1946a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines MemoryRangeFreed(thr, pc, (uptr)p, sz); 1956a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines} 1966a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines 1977ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz) { 1987ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany void *p2 = 0; 1997ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // FIXME: Handle "shrinking" more efficiently, 2007ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // it seems that some software actually does this. 2017ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (sz) { 2027ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany p2 = user_alloc(thr, pc, sz); 203efd958213d70188ae6f79afd79fe2c84956d24ffDmitry Vyukov if (p2 == 0) 204efd958213d70188ae6f79afd79fe2c84956d24ffDmitry Vyukov return 0; 2057ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (p) { 2066a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines uptr oldsz = user_alloc_usable_size(p); 2076a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines internal_memcpy(p2, p, min(oldsz, sz)); 2087ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 2097ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 210f51c3860ce0a1ae81d0dc9da27db0693718db18eDmitry Vyukov if (p) 2117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany user_free(thr, pc, p); 2127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return p2; 2137ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2147ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2156a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hinesuptr user_alloc_usable_size(const void *p) { 2168a6b5e551ab8a331f7e7bc4bfcd74926fbffa3f6Alexey Samsonov if (p == 0) 2178a6b5e551ab8a331f7e7bc4bfcd74926fbffa3f6Alexey Samsonov return 0; 2186a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines MBlock *b = ctx->metamap.GetBlock((uptr)p); 219c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar if (!b) 220c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar return 0; // Not a valid pointer. 221c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar if (b->siz == 0) 222c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar return 1; // Zero-sized allocations are actually 1 byte. 223c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar return b->siz; 2247ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2257ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2264f0ea398bdc99a4a32402057c23bbcc6d19a8eb4Alexey Samsonovvoid invoke_malloc_hook(void *ptr, uptr size) { 2274f0ea398bdc99a4a32402057c23bbcc6d19a8eb4Alexey Samsonov ThreadState *thr = cur_thread(); 2282d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (ctx == 0 || !ctx->initialized || thr->ignore_interceptors) 2294f0ea398bdc99a4a32402057c23bbcc6d19a8eb4Alexey Samsonov return; 2306a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines __sanitizer_malloc_hook(ptr, size); 231c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar RunMallocHooks(ptr, size); 2324f0ea398bdc99a4a32402057c23bbcc6d19a8eb4Alexey Samsonov} 2334f0ea398bdc99a4a32402057c23bbcc6d19a8eb4Alexey Samsonov 2344f0ea398bdc99a4a32402057c23bbcc6d19a8eb4Alexey Samsonovvoid invoke_free_hook(void *ptr) { 2354f0ea398bdc99a4a32402057c23bbcc6d19a8eb4Alexey Samsonov ThreadState *thr = cur_thread(); 2362d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (ctx == 0 || !ctx->initialized || thr->ignore_interceptors) 2374f0ea398bdc99a4a32402057c23bbcc6d19a8eb4Alexey Samsonov return; 2386a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines __sanitizer_free_hook(ptr); 239c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar RunFreeHooks(ptr); 2404f0ea398bdc99a4a32402057c23bbcc6d19a8eb4Alexey Samsonov} 2414f0ea398bdc99a4a32402057c23bbcc6d19a8eb4Alexey Samsonov 2427ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid *internal_alloc(MBlockType typ, uptr sz) { 2437ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ThreadState *thr = cur_thread(); 2449ad7c32720dfa1287f8cfd481e5d583435178caeDmitry Vyukov if (thr->nomalloc) { 2459ad7c32720dfa1287f8cfd481e5d583435178caeDmitry Vyukov thr->nomalloc = 0; // CHECK calls internal_malloc(). 2469ad7c32720dfa1287f8cfd481e5d583435178caeDmitry Vyukov CHECK(0); 2479ad7c32720dfa1287f8cfd481e5d583435178caeDmitry Vyukov } 248c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar return InternalAlloc(sz, &thr->proc()->internal_alloc_cache); 2497ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2507ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2517ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid internal_free(void *p) { 2527ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ThreadState *thr = cur_thread(); 2539ad7c32720dfa1287f8cfd481e5d583435178caeDmitry Vyukov if (thr->nomalloc) { 2549ad7c32720dfa1287f8cfd481e5d583435178caeDmitry Vyukov thr->nomalloc = 0; // CHECK calls internal_malloc(). 2559ad7c32720dfa1287f8cfd481e5d583435178caeDmitry Vyukov CHECK(0); 2569ad7c32720dfa1287f8cfd481e5d583435178caeDmitry Vyukov } 257c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar InternalFree(p, &thr->proc()->internal_alloc_cache); 2587ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2597ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2607ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} // namespace __tsan 26112530820a4d45e799cdbb83e7b3b1e8ef111650cDmitry Vyukov 26212530820a4d45e799cdbb83e7b3b1e8ef111650cDmitry Vyukovusing namespace __tsan; 26312530820a4d45e799cdbb83e7b3b1e8ef111650cDmitry Vyukov 26412530820a4d45e799cdbb83e7b3b1e8ef111650cDmitry Vyukovextern "C" { 2656a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hinesuptr __sanitizer_get_current_allocated_bytes() { 2666a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines uptr stats[AllocatorStatCount]; 267bdd844cb41718c27ef727a99a236191bc29a3df8Dmitry Vyukov allocator()->GetStats(stats); 2686a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines return stats[AllocatorStatAllocated]; 2696a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines} 27012530820a4d45e799cdbb83e7b3b1e8ef111650cDmitry Vyukov 2716a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hinesuptr __sanitizer_get_heap_size() { 2726a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines uptr stats[AllocatorStatCount]; 273bdd844cb41718c27ef727a99a236191bc29a3df8Dmitry Vyukov allocator()->GetStats(stats); 2746a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines return stats[AllocatorStatMapped]; 2756a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines} 27612530820a4d45e799cdbb83e7b3b1e8ef111650cDmitry Vyukov 2776a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hinesuptr __sanitizer_get_free_bytes() { 2781acfa02e08e6d3316c80f52255cbb7d9e3164128Dmitry Vyukov return 1; 27912530820a4d45e799cdbb83e7b3b1e8ef111650cDmitry Vyukov} 28012530820a4d45e799cdbb83e7b3b1e8ef111650cDmitry Vyukov 2816a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hinesuptr __sanitizer_get_unmapped_bytes() { 2821acfa02e08e6d3316c80f52255cbb7d9e3164128Dmitry Vyukov return 1; 28312530820a4d45e799cdbb83e7b3b1e8ef111650cDmitry Vyukov} 28412530820a4d45e799cdbb83e7b3b1e8ef111650cDmitry Vyukov 2856a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hinesuptr __sanitizer_get_estimated_allocated_size(uptr size) { 28612530820a4d45e799cdbb83e7b3b1e8ef111650cDmitry Vyukov return size; 28712530820a4d45e799cdbb83e7b3b1e8ef111650cDmitry Vyukov} 28812530820a4d45e799cdbb83e7b3b1e8ef111650cDmitry Vyukov 2896a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hinesint __sanitizer_get_ownership(const void *p) { 290bdd844cb41718c27ef727a99a236191bc29a3df8Dmitry Vyukov return allocator()->GetBlockBegin(p) != 0; 29112530820a4d45e799cdbb83e7b3b1e8ef111650cDmitry Vyukov} 29212530820a4d45e799cdbb83e7b3b1e8ef111650cDmitry Vyukov 2936a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hinesuptr __sanitizer_get_allocated_size(const void *p) { 2946a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines return user_alloc_usable_size(p); 2956a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines} 2963ce21701b4e1026a4f5157ff9771c533f4e3ef55Dmitry Vyukov 2973ce21701b4e1026a4f5157ff9771c533f4e3ef55Dmitry Vyukovvoid __tsan_on_thread_idle() { 2983ce21701b4e1026a4f5157ff9771c533f4e3ef55Dmitry Vyukov ThreadState *thr = cur_thread(); 299c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar allocator()->SwallowCache(&thr->proc()->alloc_cache); 300c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar internal_allocator()->SwallowCache(&thr->proc()->internal_alloc_cache); 301c58a43648cd6121c51a2e795a28e2ef90d7813e6Pirama Arumuga Nainar ctx->metamap.OnProcIdle(thr->proc()); 3023ce21701b4e1026a4f5157ff9771c533f4e3ef55Dmitry Vyukov} 30312530820a4d45e799cdbb83e7b3b1e8ef111650cDmitry Vyukov} // extern "C" 304