17ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany//===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===// 27ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 37ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// The LLVM Compiler Infrastructure 47ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 57ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// This file is distributed under the University of Illinois Open Source 67ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// License. See LICENSE.TXT for details. 77ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 87ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany//===----------------------------------------------------------------------===// 97ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 107ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// This file is a part of ThreadSanitizer (TSan), a race detector. 117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// Main internal TSan header file. 137ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 147ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// Ground rules: 157ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// - C++ run-time should not be used (static CTORs, RTTI, exceptions, static 167ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// function-scope locals) 177ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// - All functions/classes/etc reside in namespace __tsan, except for those 187ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// declared in tsan_interface.h. 197ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// - Platform-specific files should be used instead of ifdefs (*). 207ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// - No system headers included in header files (*). 217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// - Platform specific headres included only into platform-specific files (*). 227ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 237ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// (*) Except when inlining is critical for performance. 247ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany//===----------------------------------------------------------------------===// 257ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 267ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#ifndef TSAN_RTL_H 277ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#define TSAN_RTL_H 287ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 299edf7508e4ad536d26a202224548081d36eddd70Alexey Samsonov#include "sanitizer_common/sanitizer_common.h" 302e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov#include "sanitizer_common/sanitizer_allocator64.h" 317ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_clock.h" 327ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_defs.h" 337ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_flags.h" 347ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_sync.h" 357ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_trace.h" 367ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_vector.h" 377ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_report.h" 387ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 397ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanynamespace __tsan { 407ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 412e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov// Descriptor of user's memory block. 422e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukovstruct MBlock { 4326af89330051837bab68dcf25cf669c194b4e310Dmitry Vyukov Mutex mtx; 442e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov uptr size; 45ff35f1d82b4f145b3477ef27a7a2e7b63c486988Dmitry Vyukov u32 alloc_tid; 46ff35f1d82b4f145b3477ef27a7a2e7b63c486988Dmitry Vyukov u32 alloc_stack_id; 4726af89330051837bab68dcf25cf669c194b4e310Dmitry Vyukov SyncVar *head; 482e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov}; 492e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov 502e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov#ifndef TSAN_GO 512e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov#if defined(TSAN_COMPAT_SHADOW) && TSAN_COMPAT_SHADOW 52eee7f733f047bbd58b0787e6c492fd1bed375cc0Dmitry Vyukovconst uptr kAllocatorSpace = 0x7d0000000000ULL; 532e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov#else 542e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukovconst uptr kAllocatorSpace = 0x7d0000000000ULL; 552e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov#endif 562e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukovconst uptr kAllocatorSize = 0x10000000000ULL; // 1T. 572e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov 582e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukovtypedef SizeClassAllocator64<kAllocatorSpace, kAllocatorSize, sizeof(MBlock), 592e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov DefaultSizeClassMap> PrimaryAllocator; 602e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukovtypedef SizeClassAllocatorLocalCache<PrimaryAllocator::kNumClasses, 612e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov PrimaryAllocator> AllocatorCache; 622e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukovtypedef LargeMmapAllocator SecondaryAllocator; 632e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukovtypedef CombinedAllocator<PrimaryAllocator, AllocatorCache, 642e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov SecondaryAllocator> Allocator; 65ff35f1d82b4f145b3477ef27a7a2e7b63c486988Dmitry VyukovAllocator *allocator(); 662e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov#endif 672e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov 687fdcdf5a3a7090ec86d426a815bb42f3d35adf13Alexey Samsonovvoid TsanPrintf(const char *format, ...); 697ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 707ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// FastState (from most significant bit): 71069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov// unused : 1 727ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// tid : kTidBits 737ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// epoch : kClkBits 74069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov// unused : - 757ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// ignore_bit : 1 767ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyclass FastState { 777ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany public: 787ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany FastState(u64 tid, u64 epoch) { 79069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov x_ = tid << kTidShift; 80069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov x_ |= epoch << kClkShift; 81069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov DCHECK(tid == this->tid()); 82069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov DCHECK(epoch == this->epoch()); 837ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 847ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 857ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany explicit FastState(u64 x) 867ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany : x_(x) { 877ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 887ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 89332c62b52b3603be872b28bd3ea5e739aa28cd05Dmitry Vyukov u64 raw() const { 90332c62b52b3603be872b28bd3ea5e739aa28cd05Dmitry Vyukov return x_; 91332c62b52b3603be872b28bd3ea5e739aa28cd05Dmitry Vyukov } 92332c62b52b3603be872b28bd3ea5e739aa28cd05Dmitry Vyukov 937ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 tid() const { 94069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov u64 res = x_ >> kTidShift; 957ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return res; 967ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 97069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov 987ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 epoch() const { 99069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov u64 res = (x_ << (kTidBits + 1)) >> (64 - kClkBits); 1007ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return res; 101069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov } 102069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov 1037ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany void IncrementEpoch() { 104069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov u64 old_epoch = epoch(); 105069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov x_ += 1 << kClkShift; 106e784ad46359c898e48d0262f28a1b7b4a512f1d3Dmitry Vyukov DCHECK_EQ(old_epoch + 1, epoch()); 107069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov (void)old_epoch; 1087ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 109069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov 110069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov void SetIgnoreBit() { x_ |= kIgnoreBit; } 111069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov void ClearIgnoreBit() { x_ &= ~kIgnoreBit; } 112adfb65039646774f0f063b538f8fb0aec021f42bDmitry Vyukov bool GetIgnoreBit() const { return x_ & kIgnoreBit; } 1137ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1147ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany private: 1157ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany friend class Shadow; 116069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov static const int kTidShift = 64 - kTidBits - 1; 117069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov static const int kClkShift = kTidShift - kClkBits; 118069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov static const u64 kIgnoreBit = 1ull; 119069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov static const u64 kFreedBit = 1ull << 63; 1207ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 x_; 1217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany}; 1227ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1237ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// Shadow (from most significant bit): 124069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov// freed : 1 1257ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// tid : kTidBits 1267ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// epoch : kClkBits 1277ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// is_write : 1 1287ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// size_log : 2 1297ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// addr0 : 3 130e96366613372e616fc6f2209b81b776f931a2c58Dmitry Vyukovclass Shadow : public FastState { 1317ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany public: 1327ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany explicit Shadow(u64 x) : FastState(x) { } 1337ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1347ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany explicit Shadow(const FastState &s) : FastState(s.x_) { } 1357ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1367ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany void SetAddr0AndSizeLog(u64 addr0, unsigned kAccessSizeLog) { 1377ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK_EQ(x_ & 31, 0); 1387ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK_LE(addr0, 7); 1397ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK_LE(kAccessSizeLog, 3); 1407ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany x_ |= (kAccessSizeLog << 3) | addr0; 1417ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK_EQ(kAccessSizeLog, size_log()); 1427ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK_EQ(addr0, this->addr0()); 1437ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1447ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1457ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany void SetWrite(unsigned kAccessIsWrite) { 1467ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK_EQ(x_ & 32, 0); 1477ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (kAccessIsWrite) 1487ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany x_ |= 32; 1497ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK_EQ(kAccessIsWrite, is_write()); 1507ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1517ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1527ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany bool IsZero() const { return x_ == 0; } 1537ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 154adfb65039646774f0f063b538f8fb0aec021f42bDmitry Vyukov static inline bool TidsAreEqual(const Shadow s1, const Shadow s2) { 155069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov u64 shifted_xor = (s1.x_ ^ s2.x_) >> kTidShift; 1567ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK_EQ(shifted_xor == 0, s1.tid() == s2.tid()); 1577ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return shifted_xor == 0; 1587ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 159adfb65039646774f0f063b538f8fb0aec021f42bDmitry Vyukov 160adfb65039646774f0f063b538f8fb0aec021f42bDmitry Vyukov static inline bool Addr0AndSizeAreEqual(const Shadow s1, const Shadow s2) { 1617ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 masked_xor = (s1.x_ ^ s2.x_) & 31; 1627ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return masked_xor == 0; 1637ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1647ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1657ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany static inline bool TwoRangesIntersect(Shadow s1, Shadow s2, 1667ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany unsigned kS2AccessSize) { 1677ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany bool res = false; 1687ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 diff = s1.addr0() - s2.addr0(); 1697ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if ((s64)diff < 0) { // s1.addr0 < s2.addr0 // NOLINT 1707ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // if (s1.addr0() + size1) > s2.addr0()) return true; 1717ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (s1.size() > -diff) res = true; 1727ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } else { 1737ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // if (s2.addr0() + kS2AccessSize > s1.addr0()) return true; 1747ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (kS2AccessSize > diff) res = true; 1757ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1767ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK_EQ(res, TwoRangesIntersectSLOW(s1, s2)); 1777ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK_EQ(res, TwoRangesIntersectSLOW(s2, s1)); 1787ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return res; 1797ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1807ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1817ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // The idea behind the offset is as follows. 1827ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Consider that we have 8 bool's contained within a single 8-byte block 1837ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // (mapped to a single shadow "cell"). Now consider that we write to the bools 1847ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // from a single thread (which we consider the common case). 1857ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // W/o offsetting each access will have to scan 4 shadow values at average 1867ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // to find the corresponding shadow value for the bool. 1877ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // With offsetting we start scanning shadow with the offset so that 1887ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // each access hits necessary shadow straight off (at least in an expected 1897ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // optimistic case). 1907ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // This logic works seamlessly for any layout of user data. For example, 1917ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // if user data is {int, short, char, char}, then accesses to the int are 1927ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // offsetted to 0, short - 4, 1st char - 6, 2nd char - 7. Hopefully, accesses 1937ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // from a single thread won't need to scan all 8 shadow values. 1947ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany unsigned ComputeSearchOffset() { 1957ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return x_ & 7; 1967ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1977ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 addr0() const { return x_ & 7; } 1987ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 size() const { return 1ull << size_log(); } 1997ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany bool is_write() const { return x_ & 32; } 2007ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 201069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov // The idea behind the freed bit is as follows. 202069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov // When the memory is freed (or otherwise unaccessible) we write to the shadow 203069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov // values with tid/epoch related to the free and the freed bit set. 204069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov // During memory accesses processing the freed bit is considered 205069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov // as msb of tid. So any access races with shadow with freed bit set 206069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov // (it is as if write from a thread with which we never synchronized before). 207069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov // This allows us to detect accesses to freed memory w/o additional 208069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov // overheads in memory access processing and at the same time restore 209069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov // tid/epoch of free. 210069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov void MarkAsFreed() { 211069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov x_ |= kFreedBit; 212069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov } 213069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov 214069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov bool GetFreedAndReset() { 215069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov bool res = x_ & kFreedBit; 216069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov x_ &= ~kFreedBit; 217069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov return res; 218069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov } 219069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov 2207ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany private: 2217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 size_log() const { return (x_ >> 3) & 3; } 222adfb65039646774f0f063b538f8fb0aec021f42bDmitry Vyukov 223adfb65039646774f0f063b538f8fb0aec021f42bDmitry Vyukov static bool TwoRangesIntersectSLOW(const Shadow s1, const Shadow s2) { 224adfb65039646774f0f063b538f8fb0aec021f42bDmitry Vyukov if (s1.addr0() == s2.addr0()) return true; 225adfb65039646774f0f063b538f8fb0aec021f42bDmitry Vyukov if (s1.addr0() < s2.addr0() && s1.addr0() + s1.size() > s2.addr0()) 226adfb65039646774f0f063b538f8fb0aec021f42bDmitry Vyukov return true; 227adfb65039646774f0f063b538f8fb0aec021f42bDmitry Vyukov if (s2.addr0() < s1.addr0() && s2.addr0() + s2.size() > s1.addr0()) 228adfb65039646774f0f063b538f8fb0aec021f42bDmitry Vyukov return true; 229adfb65039646774f0f063b538f8fb0aec021f42bDmitry Vyukov return false; 230adfb65039646774f0f063b538f8fb0aec021f42bDmitry Vyukov } 2317ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany}; 2327ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2337ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// Freed memory. 2347ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// As if 8-byte write by thread 0xff..f at epoch 0xff..f, races with everything. 2357ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyconst u64 kShadowFreed = 0xfffffffffffffff8ull; 2367ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 237e96366613372e616fc6f2209b81b776f931a2c58Dmitry Vyukovstruct SignalContext; 2387ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2397ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// This struct is stored in TLS. 2407ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystruct ThreadState { 2417ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany FastState fast_state; 2427ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Synch epoch represents the threads's epoch before the last synchronization 2437ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // action. It allows to reduce number of shadow state updates. 2447ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // For example, fast_synch_epoch=100, last write to addr X was at epoch=150, 2457ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // if we are processing write to X from the same thread at epoch=200, 2467ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // we do nothing, because both writes happen in the same 'synch epoch'. 2477ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // That is, if another memory access does not race with the former write, 2487ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // it does not race with the latter as well. 2497ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // QUESTION: can we can squeeze this into ThreadState::Fast? 2507ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // E.g. ThreadState::Fast is a 44-bit, 32 are taken by synch_epoch and 12 are 2517ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // taken by epoch between synchs. 2527ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // This way we can save one load from tls. 2537ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 fast_synch_epoch; 2547ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read. 2557ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // We do not distinguish beteween ignoring reads and writes 2567ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // for better performance. 2577ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int ignore_reads_and_writes; 2587ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr *shadow_stack_pos; 2597ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 *racy_shadow_addr; 2607ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 racy_state[2]; 2617ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Trace trace; 26225d1c799087af5757ab6efc4a77558565fb1744aDmitry Vyukov#ifndef TSAN_GO 26325d1c799087af5757ab6efc4a77558565fb1744aDmitry Vyukov // C/C++ uses embed shadow stack of fixed size. 2647ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr shadow_stack[kShadowStackSize]; 26525d1c799087af5757ab6efc4a77558565fb1744aDmitry Vyukov#else 26625d1c799087af5757ab6efc4a77558565fb1744aDmitry Vyukov // Go uses satellite shadow stack with dynamic size. 26725d1c799087af5757ab6efc4a77558565fb1744aDmitry Vyukov uptr *shadow_stack; 26825d1c799087af5757ab6efc4a77558565fb1744aDmitry Vyukov uptr *shadow_stack_end; 26925d1c799087af5757ab6efc4a77558565fb1744aDmitry Vyukov#endif 2707ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ThreadClock clock; 2712e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov#ifndef TSAN_GO 2722e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov AllocatorCache alloc_cache; 2732e87051d136db3150a3ca322d8862f92b0a684bbDmitry Vyukov#endif 2747ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 stat[StatCnt]; 2757ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany const int tid; 276ff35f1d82b4f145b3477ef27a7a2e7b63c486988Dmitry Vyukov const int unique_id; 2777ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int in_rtl; 2781fc03d50a87302689482efa24b045a393097b6c3Dmitry Vyukov bool is_alive; 2797ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany const uptr stk_addr; 2807ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany const uptr stk_size; 2817ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany const uptr tls_addr; 2827ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany const uptr tls_size; 2837ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2847ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DeadlockDetector deadlock_detector; 2857ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2867ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany bool in_signal_handler; 287e96366613372e616fc6f2209b81b776f931a2c58Dmitry Vyukov SignalContext *signal_ctx; 288e96366613372e616fc6f2209b81b776f931a2c58Dmitry Vyukov 289848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukov#ifndef TSAN_GO 290848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukov u32 last_sleep_stack_id; 291848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukov ThreadClock last_sleep_clock; 292848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukov#endif 293848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukov 2949ad7c32720dfa1287f8cfd481e5d583435178caeDmitry Vyukov // Set in regions of runtime that must be signal-safe and fork-safe. 2959ad7c32720dfa1287f8cfd481e5d583435178caeDmitry Vyukov // If set, malloc must not be called. 2969ad7c32720dfa1287f8cfd481e5d583435178caeDmitry Vyukov int nomalloc; 2977ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 298ff35f1d82b4f145b3477ef27a7a2e7b63c486988Dmitry Vyukov explicit ThreadState(Context *ctx, int tid, int unique_id, u64 epoch, 2997ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr stk_addr, uptr stk_size, 3007ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr tls_addr, uptr tls_size); 3017ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany}; 3027ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3037ac41484ea322e0ea5774df681660269f5dc321eKostya SerebryanyContext *CTX(); 3047ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 305b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov#ifndef TSAN_GO 306b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukovextern THREADLOCAL char cur_thread_placeholder[]; 3077ac41484ea322e0ea5774df681660269f5dc321eKostya SerebryanyINLINE ThreadState *cur_thread() { 3087ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return reinterpret_cast<ThreadState *>(&cur_thread_placeholder); 3097ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 310b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov#endif 3117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyenum ThreadStatus { 3137ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ThreadStatusInvalid, // Non-existent thread, data is invalid. 3147ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ThreadStatusCreated, // Created but not yet running. 3157ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ThreadStatusRunning, // The thread is currently running. 3167ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ThreadStatusFinished, // Joinable thread is finished but not yet joined. 3177ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ThreadStatusDead, // Joined, but some info (trace) is still alive. 3187ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany}; 3197ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3207ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// An info about a thread that is hold for some time after its termination. 3217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystruct ThreadDeadInfo { 3227ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Trace trace; 3237ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany}; 3247ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3257ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystruct ThreadContext { 3267ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany const int tid; 3277ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int unique_id; // Non-rolling thread id. 3287ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr user_id; // Some opaque user thread id (e.g. pthread_t). 3297ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ThreadState *thr; 3307ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ThreadStatus status; 3317ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany bool detached; 3327ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int reuse_count; 3337ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany SyncClock sync; 3347ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Epoch at which the thread had started. 3357ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // If we see an event from the thread stamped by an older epoch, 3367ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // the event is from a dead thread that shared tid with this thread. 3377ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 epoch0; 3387ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 epoch1; 3397ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StackTrace creation_stack; 3409d2ffc2ee08216f8fad9b1bd267d1f112e0d2f01Dmitry Vyukov ThreadDeadInfo *dead_info; 3419d2ffc2ee08216f8fad9b1bd267d1f112e0d2f01Dmitry Vyukov ThreadContext *dead_next; // In dead thread list. 3427ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3437ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany explicit ThreadContext(int tid); 3447ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany}; 3457ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3467ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystruct RacyStacks { 3477ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany MD5Hash hash[2]; 3487ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany bool operator==(const RacyStacks &other) const { 3497ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (hash[0] == other.hash[0] && hash[1] == other.hash[1]) 3507ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return true; 3517ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (hash[0] == other.hash[1] && hash[1] == other.hash[0]) 3527ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return true; 3537ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return false; 3547ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 3557ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany}; 3567ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3577ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystruct RacyAddress { 3587ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr addr_min; 3597ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr addr_max; 3607ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany}; 3617ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3627ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystruct Context { 3637ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Context(); 3647ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3657ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany bool initialized; 3667ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3677ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany SyncTab synctab; 3687ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3697ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Mutex report_mtx; 3707ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int nreported; 3717ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int nmissed_expected; 3727ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3737ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Mutex thread_mtx; 3743d6ae1580e5f34f02d24ff0c8bb352a5026c327aKostya Serebryany unsigned thread_seq; 3753d6ae1580e5f34f02d24ff0c8bb352a5026c327aKostya Serebryany unsigned unique_thread_seq; 3767ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int alive_threads; 3777ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int max_alive_threads; 3787ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ThreadContext *threads[kMaxTid]; 3797ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int dead_list_size; 3807ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ThreadContext* dead_list_head; 3817ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ThreadContext* dead_list_tail; 3827ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3837ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Vector<RacyStacks> racy_stacks; 3847ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Vector<RacyAddress> racy_addresses; 3857ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3867ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Flags flags; 3877ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3887ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 stat[StatCnt]; 3897ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 int_alloc_cnt[MBlockTypeCount]; 3907ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 int_alloc_siz[MBlockTypeCount]; 3917ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany}; 3927ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3937ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyclass ScopedInRtl { 3947ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany public: 3957ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ScopedInRtl(); 3967ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ~ScopedInRtl(); 3977ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany private: 3987ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ThreadState*thr_; 3997ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int in_rtl_; 4007ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int errno_; 4017ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany}; 4027ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 4037ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyclass ScopedReport { 4047ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany public: 4057ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany explicit ScopedReport(ReportType typ); 4067ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ~ScopedReport(); 4077ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 4087ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany void AddStack(const StackTrace *stack); 4097ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany void AddMemoryAccess(uptr addr, Shadow s, const StackTrace *stack); 4107ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany void AddThread(const ThreadContext *tctx); 4117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany void AddMutex(const SyncVar *s); 4127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany void AddLocation(uptr addr, uptr size); 413848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukov void AddSleep(u32 stack_id); 4147ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 4157ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany const ReportDesc *GetReport() const; 4167ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 4177ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany private: 4187ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Context *ctx_; 4197ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ReportDesc *rep_; 4207ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 4217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ScopedReport(const ScopedReport&); 4227ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany void operator = (const ScopedReport&); 4237ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany}; 4247ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 425332c62b52b3603be872b28bd3ea5e739aa28cd05Dmitry Vyukovvoid RestoreStack(int tid, const u64 epoch, StackTrace *stk); 426332c62b52b3603be872b28bd3ea5e739aa28cd05Dmitry Vyukov 4277ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid StatAggregate(u64 *dst, u64 *src); 4287ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid StatOutput(u64 *stat); 4297ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid ALWAYS_INLINE INLINE StatInc(ThreadState *thr, StatType typ, u64 n = 1) { 4307ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (kCollectStats) 4317ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->stat[typ] += n; 4327ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 4337ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 4347ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid InitializeShadowMemory(); 4357ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid InitializeInterceptors(); 4367ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid InitializeDynamicAnnotations(); 4377ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 4387ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid ReportRace(ThreadState *thr); 439f5820e74ad31eb8352049c880f8d58e286a9b713Dmitry Vyukovbool OutputReport(const ScopedReport &srep, 440f5820e74ad31eb8352049c880f8d58e286a9b713Dmitry Vyukov const ReportStack *suppress_stack = 0); 4417ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanybool IsExpectedReport(uptr addr, uptr size); 4427ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 4437ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1 44467a64dd8259fdbd867633b27f54d584f435f1ce6Alexey Samsonov# define DPrintf TsanPrintf 4457ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#else 4467ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany# define DPrintf(...) 4477ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 4487ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 4497ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 2 45067a64dd8259fdbd867633b27f54d584f435f1ce6Alexey Samsonov# define DPrintf2 TsanPrintf 4517ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#else 4527ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany# define DPrintf2(...) 4537ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 4547ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 455848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukovu32 CurrentStackId(ThreadState *thr, uptr pc); 4561da1056127d1dbcacdd035eb4149257848f7c4dfDmitry Vyukovvoid PrintCurrentStack(ThreadState *thr, uptr pc); 457848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukov 4587ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid Initialize(ThreadState *thr); 4597ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyint Finalize(ThreadState *thr); 4607ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 4617ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MemoryAccess(ThreadState *thr, uptr pc, uptr addr, 4627ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int kAccessSizeLog, bool kAccessIsWrite); 4637ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MemoryAccessImpl(ThreadState *thr, uptr addr, 4647ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int kAccessSizeLog, bool kAccessIsWrite, FastState fast_state, 4657ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 *shadow_mem, Shadow cur); 4667ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MemoryRead1Byte(ThreadState *thr, uptr pc, uptr addr); 4677ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MemoryWrite1Byte(ThreadState *thr, uptr pc, uptr addr); 4687ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MemoryRead8Byte(ThreadState *thr, uptr pc, uptr addr); 4697ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MemoryWrite8Byte(ThreadState *thr, uptr pc, uptr addr); 4707ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, 4717ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr size, bool is_write); 4727ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size); 4737ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size); 47426af89330051837bab68dcf25cf669c194b4e310Dmitry Vyukovvoid MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size); 4757ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid IgnoreCtl(ThreadState *thr, bool write, bool begin); 4767ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 4777ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid FuncEntry(ThreadState *thr, uptr pc); 4787ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid FuncExit(ThreadState *thr); 4797ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 4807ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyint ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached); 4817ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid ThreadStart(ThreadState *thr, int tid); 4827ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid ThreadFinish(ThreadState *thr); 4837ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyint ThreadTid(ThreadState *thr, uptr pc, uptr uid); 4847ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid ThreadJoin(ThreadState *thr, uptr pc, int tid); 4857ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid ThreadDetach(ThreadState *thr, uptr pc, int tid); 4867ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid ThreadFinalize(ThreadState *thr); 4878f1104cbf1af615242e14c66d1b3dd9e8437b152Dmitry Vyukovvoid ThreadFinalizerGoroutine(ThreadState *thr); 4887ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 489c20e9ba760a93706c9160133cdcce62194f9694fDmitry Vyukovvoid MutexCreate(ThreadState *thr, uptr pc, uptr addr, 490c20e9ba760a93706c9160133cdcce62194f9694fDmitry Vyukov bool rw, bool recursive, bool linker_init); 4917ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MutexDestroy(ThreadState *thr, uptr pc, uptr addr); 4927ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MutexLock(ThreadState *thr, uptr pc, uptr addr); 4937ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MutexUnlock(ThreadState *thr, uptr pc, uptr addr); 4947ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MutexReadLock(ThreadState *thr, uptr pc, uptr addr); 4957ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr); 4967ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr); 4977ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 4987ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid Acquire(ThreadState *thr, uptr pc, uptr addr); 4997ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid Release(ThreadState *thr, uptr pc, uptr addr); 5009d150bdb433ddd092073dabd87ba15aa176603a1Dmitry Vyukovvoid ReleaseStore(ThreadState *thr, uptr pc, uptr addr); 501848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukovvoid AfterSleep(ThreadState *thr, uptr pc); 5027ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 5037ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// The hacky call uses custom calling convention and an assembly thunk. 5047ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// It is considerably faster that a normal call for the caller 5057ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// if it is not executed (it is intended for slow paths from hot functions). 5067ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// The trick is that the call preserves all registers and the compiler 5077ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// does not treat it as a call. 5087ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// If it does not work for you, use normal call. 5097ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#if TSAN_DEBUG == 0 5107ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// The caller may not create the stack frame for itself at all, 5117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// so we create a reserve stack frame for it (1024b must be enough). 5127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#define HACKY_CALL(f) \ 51341e8153c4a29b28bc7c80d35bc1d43438acdd4e7Dmitry Vyukov __asm__ __volatile__("sub $1024, %%rsp;" \ 51441e8153c4a29b28bc7c80d35bc1d43438acdd4e7Dmitry Vyukov "/*.cfi_adjust_cfa_offset 1024;*/" \ 5157ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany "call " #f "_thunk;" \ 51641e8153c4a29b28bc7c80d35bc1d43438acdd4e7Dmitry Vyukov "add $1024, %%rsp;" \ 51741e8153c4a29b28bc7c80d35bc1d43438acdd4e7Dmitry Vyukov "/*.cfi_adjust_cfa_offset -1024;*/" \ 51841e8153c4a29b28bc7c80d35bc1d43438acdd4e7Dmitry Vyukov ::: "memory", "cc"); 5197ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#else 5207ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#define HACKY_CALL(f) f() 5217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 5227ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 523b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukovvoid TraceSwitch(ThreadState *thr); 524b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov 5257ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyextern "C" void __tsan_trace_switch(); 5267ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid ALWAYS_INLINE INLINE TraceAddEvent(ThreadState *thr, u64 epoch, 5277ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany EventType typ, uptr addr) { 5287ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatEvents); 529b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov if (UNLIKELY((epoch % kTracePartSize) == 0)) { 530b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov#ifndef TSAN_GO 5317ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany HACKY_CALL(__tsan_trace_switch); 532b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov#else 533b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov TraceSwitch(thr); 534b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov#endif 535b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov } 5367ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Event *evp = &thr->trace.events[epoch % kTraceSize]; 5377ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Event ev = (u64)addr | ((u64)typ << 61); 5387ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany *evp = ev; 5397ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 5407ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 5417ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} // namespace __tsan 5427ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 5437ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif // TSAN_RTL_H 544