tsan_rtl.h revision 0dc47b652dfbe0d61d153eded02bae9487a7b539
1//===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file is a part of ThreadSanitizer (TSan), a race detector. 11// 12// Main internal TSan header file. 13// 14// Ground rules: 15// - C++ run-time should not be used (static CTORs, RTTI, exceptions, static 16// function-scope locals) 17// - All functions/classes/etc reside in namespace __tsan, except for those 18// declared in tsan_interface.h. 19// - Platform-specific files should be used instead of ifdefs (*). 20// - No system headers included in header files (*). 21// - Platform specific headres included only into platform-specific files (*). 22// 23// (*) Except when inlining is critical for performance. 24//===----------------------------------------------------------------------===// 25 26#ifndef TSAN_RTL_H 27#define TSAN_RTL_H 28 29#include "sanitizer_common/sanitizer_allocator.h" 30#include "sanitizer_common/sanitizer_common.h" 31#include "sanitizer_common/sanitizer_thread_registry.h" 32#include "tsan_clock.h" 33#include "tsan_defs.h" 34#include "tsan_flags.h" 35#include "tsan_sync.h" 36#include "tsan_trace.h" 37#include "tsan_vector.h" 38#include "tsan_report.h" 39#include "tsan_platform.h" 40#include "tsan_mutexset.h" 41 42#if SANITIZER_WORDSIZE != 64 43# error "ThreadSanitizer is supported only on 64-bit platforms" 44#endif 45 46namespace __tsan { 47 48// Descriptor of user's memory block. 49struct MBlock { 50 /* 51 u64 mtx : 1; // must be first 52 u64 lst : 44; 53 u64 stk : 31; // on word boundary 54 u64 tid : kTidBits; 55 u64 siz : 128 - 1 - 31 - 44 - kTidBits; // 39 56 */ 57 u64 raw[2]; 58 59 void Init(uptr siz, u32 tid, u32 stk) { 60 raw[0] = raw[1] = 0; 61 raw[1] |= (u64)siz << ((1 + 44 + 31 + kTidBits) % 64); 62 raw[1] |= (u64)tid << ((1 + 44 + 31) % 64); 63 raw[0] |= (u64)stk << (1 + 44); 64 raw[1] |= (u64)stk >> (64 - 44 - 1); 65 DCHECK_EQ(Size(), siz); 66 DCHECK_EQ(Tid(), tid); 67 DCHECK_EQ(StackId(), stk); 68 } 69 70 u32 Tid() const { 71 return GetLsb(raw[1] >> ((1 + 44 + 31) % 64), kTidBits); 72 } 73 74 uptr Size() const { 75 return raw[1] >> ((1 + 31 + 44 + kTidBits) % 64); 76 } 77 78 u32 StackId() const { 79 return (raw[0] >> (1 + 44)) | GetLsb(raw[1] << (64 - 44 - 1), 31); 80 } 81 82 SyncVar *ListHead() const { 83 return (SyncVar*)(GetLsb(raw[0] >> 1, 44) << 3); 84 } 85 86 void ListPush(SyncVar *v) { 87 SyncVar *lst = ListHead(); 88 v->next = lst; 89 u64 x = (u64)v ^ (u64)lst; 90 x = (x >> 3) << 1; 91 raw[0] ^= x; 92 DCHECK_EQ(ListHead(), v); 93 } 94 95 SyncVar *ListPop() { 96 SyncVar *lst = ListHead(); 97 SyncVar *nxt = lst->next; 98 lst->next = 0; 99 u64 x = (u64)lst ^ (u64)nxt; 100 x = (x >> 3) << 1; 101 raw[0] ^= x; 102 DCHECK_EQ(ListHead(), nxt); 103 return lst; 104 } 105 106 void ListReset() { 107 SyncVar *lst = ListHead(); 108 u64 x = (u64)lst; 109 x = (x >> 3) << 1; 110 raw[0] ^= x; 111 DCHECK_EQ(ListHead(), 0); 112 } 113 114 void Lock(); 115 void Unlock(); 116 typedef GenericScopedLock<MBlock> ScopedLock; 117}; 118 119#ifndef TSAN_GO 120#if defined(TSAN_COMPAT_SHADOW) && TSAN_COMPAT_SHADOW 121const uptr kAllocatorSpace = 0x7d0000000000ULL; 122#else 123const uptr kAllocatorSpace = 0x7d0000000000ULL; 124#endif 125const uptr kAllocatorSize = 0x10000000000ULL; // 1T. 126 127struct MapUnmapCallback; 128typedef SizeClassAllocator64<kAllocatorSpace, kAllocatorSize, sizeof(MBlock), 129 DefaultSizeClassMap, MapUnmapCallback> PrimaryAllocator; 130typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; 131typedef LargeMmapAllocator<MapUnmapCallback> SecondaryAllocator; 132typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, 133 SecondaryAllocator> Allocator; 134Allocator *allocator(); 135#endif 136 137void TsanCheckFailed(const char *file, int line, const char *cond, 138 u64 v1, u64 v2); 139 140const u64 kShadowRodata = (u64)-1; // .rodata shadow marker 141 142// FastState (from most significant bit): 143// ignore : 1 144// tid : kTidBits 145// epoch : kClkBits 146// unused : - 147// history_size : 3 148class FastState { 149 public: 150 FastState(u64 tid, u64 epoch) { 151 x_ = tid << kTidShift; 152 x_ |= epoch << kClkShift; 153 DCHECK_EQ(tid, this->tid()); 154 DCHECK_EQ(epoch, this->epoch()); 155 DCHECK_EQ(GetIgnoreBit(), false); 156 } 157 158 explicit FastState(u64 x) 159 : x_(x) { 160 } 161 162 u64 raw() const { 163 return x_; 164 } 165 166 u64 tid() const { 167 u64 res = (x_ & ~kIgnoreBit) >> kTidShift; 168 return res; 169 } 170 171 u64 TidWithIgnore() const { 172 u64 res = x_ >> kTidShift; 173 return res; 174 } 175 176 u64 epoch() const { 177 u64 res = (x_ << (kTidBits + 1)) >> (64 - kClkBits); 178 return res; 179 } 180 181 void IncrementEpoch() { 182 u64 old_epoch = epoch(); 183 x_ += 1 << kClkShift; 184 DCHECK_EQ(old_epoch + 1, epoch()); 185 (void)old_epoch; 186 } 187 188 void SetIgnoreBit() { x_ |= kIgnoreBit; } 189 void ClearIgnoreBit() { x_ &= ~kIgnoreBit; } 190 bool GetIgnoreBit() const { return (s64)x_ < 0; } 191 192 void SetHistorySize(int hs) { 193 CHECK_GE(hs, 0); 194 CHECK_LE(hs, 7); 195 x_ = (x_ & ~7) | hs; 196 } 197 198 int GetHistorySize() const { 199 return (int)(x_ & 7); 200 } 201 202 void ClearHistorySize() { 203 x_ &= ~7; 204 } 205 206 u64 GetTracePos() const { 207 const int hs = GetHistorySize(); 208 // When hs == 0, the trace consists of 2 parts. 209 const u64 mask = (1ull << (kTracePartSizeBits + hs + 1)) - 1; 210 return epoch() & mask; 211 } 212 213 private: 214 friend class Shadow; 215 static const int kTidShift = 64 - kTidBits - 1; 216 static const int kClkShift = kTidShift - kClkBits; 217 static const u64 kIgnoreBit = 1ull << 63; 218 static const u64 kFreedBit = 1ull << 63; 219 u64 x_; 220}; 221 222// Shadow (from most significant bit): 223// freed : 1 224// tid : kTidBits 225// epoch : kClkBits 226// is_atomic : 1 227// is_read : 1 228// size_log : 2 229// addr0 : 3 230class Shadow : public FastState { 231 public: 232 explicit Shadow(u64 x) 233 : FastState(x) { 234 } 235 236 explicit Shadow(const FastState &s) 237 : FastState(s.x_) { 238 ClearHistorySize(); 239 } 240 241 void SetAddr0AndSizeLog(u64 addr0, unsigned kAccessSizeLog) { 242 DCHECK_EQ(x_ & 31, 0); 243 DCHECK_LE(addr0, 7); 244 DCHECK_LE(kAccessSizeLog, 3); 245 x_ |= (kAccessSizeLog << 3) | addr0; 246 DCHECK_EQ(kAccessSizeLog, size_log()); 247 DCHECK_EQ(addr0, this->addr0()); 248 } 249 250 void SetWrite(unsigned kAccessIsWrite) { 251 DCHECK_EQ(x_ & kReadBit, 0); 252 if (!kAccessIsWrite) 253 x_ |= kReadBit; 254 DCHECK_EQ(kAccessIsWrite, IsWrite()); 255 } 256 257 void SetAtomic(bool kIsAtomic) { 258 DCHECK(!IsAtomic()); 259 if (kIsAtomic) 260 x_ |= kAtomicBit; 261 DCHECK_EQ(IsAtomic(), kIsAtomic); 262 } 263 264 bool IsAtomic() const { 265 return x_ & kAtomicBit; 266 } 267 268 bool IsZero() const { 269 return x_ == 0; 270 } 271 272 static inline bool TidsAreEqual(const Shadow s1, const Shadow s2) { 273 u64 shifted_xor = (s1.x_ ^ s2.x_) >> kTidShift; 274 DCHECK_EQ(shifted_xor == 0, s1.TidWithIgnore() == s2.TidWithIgnore()); 275 return shifted_xor == 0; 276 } 277 278 static inline bool Addr0AndSizeAreEqual(const Shadow s1, const Shadow s2) { 279 u64 masked_xor = (s1.x_ ^ s2.x_) & 31; 280 return masked_xor == 0; 281 } 282 283 static inline bool TwoRangesIntersect(Shadow s1, Shadow s2, 284 unsigned kS2AccessSize) { 285 bool res = false; 286 u64 diff = s1.addr0() - s2.addr0(); 287 if ((s64)diff < 0) { // s1.addr0 < s2.addr0 // NOLINT 288 // if (s1.addr0() + size1) > s2.addr0()) return true; 289 if (s1.size() > -diff) res = true; 290 } else { 291 // if (s2.addr0() + kS2AccessSize > s1.addr0()) return true; 292 if (kS2AccessSize > diff) res = true; 293 } 294 DCHECK_EQ(res, TwoRangesIntersectSLOW(s1, s2)); 295 DCHECK_EQ(res, TwoRangesIntersectSLOW(s2, s1)); 296 return res; 297 } 298 299 // The idea behind the offset is as follows. 300 // Consider that we have 8 bool's contained within a single 8-byte block 301 // (mapped to a single shadow "cell"). Now consider that we write to the bools 302 // from a single thread (which we consider the common case). 303 // W/o offsetting each access will have to scan 4 shadow values at average 304 // to find the corresponding shadow value for the bool. 305 // With offsetting we start scanning shadow with the offset so that 306 // each access hits necessary shadow straight off (at least in an expected 307 // optimistic case). 308 // This logic works seamlessly for any layout of user data. For example, 309 // if user data is {int, short, char, char}, then accesses to the int are 310 // offsetted to 0, short - 4, 1st char - 6, 2nd char - 7. Hopefully, accesses 311 // from a single thread won't need to scan all 8 shadow values. 312 unsigned ComputeSearchOffset() { 313 return x_ & 7; 314 } 315 u64 addr0() const { return x_ & 7; } 316 u64 size() const { return 1ull << size_log(); } 317 bool IsWrite() const { return !IsRead(); } 318 bool IsRead() const { return x_ & kReadBit; } 319 320 // The idea behind the freed bit is as follows. 321 // When the memory is freed (or otherwise unaccessible) we write to the shadow 322 // values with tid/epoch related to the free and the freed bit set. 323 // During memory accesses processing the freed bit is considered 324 // as msb of tid. So any access races with shadow with freed bit set 325 // (it is as if write from a thread with which we never synchronized before). 326 // This allows us to detect accesses to freed memory w/o additional 327 // overheads in memory access processing and at the same time restore 328 // tid/epoch of free. 329 void MarkAsFreed() { 330 x_ |= kFreedBit; 331 } 332 333 bool IsFreed() const { 334 return x_ & kFreedBit; 335 } 336 337 bool GetFreedAndReset() { 338 bool res = x_ & kFreedBit; 339 x_ &= ~kFreedBit; 340 return res; 341 } 342 343 bool IsBothReadsOrAtomic(bool kIsWrite, bool kIsAtomic) const { 344 // analyzes 5-th bit (is_read) and 6-th bit (is_atomic) 345 bool v = x_ & u64(((kIsWrite ^ 1) << kReadShift) 346 | (kIsAtomic << kAtomicShift)); 347 DCHECK_EQ(v, (!IsWrite() && !kIsWrite) || (IsAtomic() && kIsAtomic)); 348 return v; 349 } 350 351 bool IsRWNotWeaker(bool kIsWrite, bool kIsAtomic) const { 352 bool v = ((x_ >> kReadShift) & 3) 353 <= u64((kIsWrite ^ 1) | (kIsAtomic << 1)); 354 DCHECK_EQ(v, (IsAtomic() < kIsAtomic) || 355 (IsAtomic() == kIsAtomic && !IsWrite() <= !kIsWrite)); 356 return v; 357 } 358 359 bool IsRWWeakerOrEqual(bool kIsWrite, bool kIsAtomic) const { 360 bool v = ((x_ >> kReadShift) & 3) 361 >= u64((kIsWrite ^ 1) | (kIsAtomic << 1)); 362 DCHECK_EQ(v, (IsAtomic() > kIsAtomic) || 363 (IsAtomic() == kIsAtomic && !IsWrite() >= !kIsWrite)); 364 return v; 365 } 366 367 private: 368 static const u64 kReadShift = 5; 369 static const u64 kReadBit = 1ull << kReadShift; 370 static const u64 kAtomicShift = 6; 371 static const u64 kAtomicBit = 1ull << kAtomicShift; 372 373 u64 size_log() const { return (x_ >> 3) & 3; } 374 375 static bool TwoRangesIntersectSLOW(const Shadow s1, const Shadow s2) { 376 if (s1.addr0() == s2.addr0()) return true; 377 if (s1.addr0() < s2.addr0() && s1.addr0() + s1.size() > s2.addr0()) 378 return true; 379 if (s2.addr0() < s1.addr0() && s2.addr0() + s2.size() > s1.addr0()) 380 return true; 381 return false; 382 } 383}; 384 385struct SignalContext; 386 387// This struct is stored in TLS. 388struct ThreadState { 389 FastState fast_state; 390 // Synch epoch represents the threads's epoch before the last synchronization 391 // action. It allows to reduce number of shadow state updates. 392 // For example, fast_synch_epoch=100, last write to addr X was at epoch=150, 393 // if we are processing write to X from the same thread at epoch=200, 394 // we do nothing, because both writes happen in the same 'synch epoch'. 395 // That is, if another memory access does not race with the former write, 396 // it does not race with the latter as well. 397 // QUESTION: can we can squeeze this into ThreadState::Fast? 398 // E.g. ThreadState::Fast is a 44-bit, 32 are taken by synch_epoch and 12 are 399 // taken by epoch between synchs. 400 // This way we can save one load from tls. 401 u64 fast_synch_epoch; 402 // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read. 403 // We do not distinguish beteween ignoring reads and writes 404 // for better performance. 405 int ignore_reads_and_writes; 406 uptr *shadow_stack_pos; 407 u64 *racy_shadow_addr; 408 u64 racy_state[2]; 409#ifndef TSAN_GO 410 // C/C++ uses embed shadow stack of fixed size. 411 uptr shadow_stack[kShadowStackSize]; 412#else 413 // Go uses satellite shadow stack with dynamic size. 414 uptr *shadow_stack; 415 uptr *shadow_stack_end; 416#endif 417 MutexSet mset; 418 ThreadClock clock; 419#ifndef TSAN_GO 420 AllocatorCache alloc_cache; 421#endif 422 u64 stat[StatCnt]; 423 const int tid; 424 const int unique_id; 425 int in_rtl; 426 bool in_symbolizer; 427 bool is_alive; 428 bool is_freeing; 429 bool is_vptr_access; 430 const uptr stk_addr; 431 const uptr stk_size; 432 const uptr tls_addr; 433 const uptr tls_size; 434 435 DeadlockDetector deadlock_detector; 436 437 bool in_signal_handler; 438 SignalContext *signal_ctx; 439 440#ifndef TSAN_GO 441 u32 last_sleep_stack_id; 442 ThreadClock last_sleep_clock; 443#endif 444 445 // Set in regions of runtime that must be signal-safe and fork-safe. 446 // If set, malloc must not be called. 447 int nomalloc; 448 449 explicit ThreadState(Context *ctx, int tid, int unique_id, u64 epoch, 450 uptr stk_addr, uptr stk_size, 451 uptr tls_addr, uptr tls_size); 452}; 453 454Context *CTX(); 455 456#ifndef TSAN_GO 457extern THREADLOCAL char cur_thread_placeholder[]; 458INLINE ThreadState *cur_thread() { 459 return reinterpret_cast<ThreadState *>(&cur_thread_placeholder); 460} 461#endif 462 463class ThreadContext : public ThreadContextBase { 464 public: 465 explicit ThreadContext(int tid); 466 ~ThreadContext(); 467 ThreadState *thr; 468#ifdef TSAN_GO 469 StackTrace creation_stack; 470#else 471 u32 creation_stack_id; 472#endif 473 SyncClock sync; 474 // Epoch at which the thread had started. 475 // If we see an event from the thread stamped by an older epoch, 476 // the event is from a dead thread that shared tid with this thread. 477 u64 epoch0; 478 u64 epoch1; 479 480 // Override superclass callbacks. 481 void OnDead(); 482 void OnJoined(void *arg); 483 void OnFinished(); 484 void OnStarted(void *arg); 485 void OnCreated(void *arg); 486 void OnReset(); 487}; 488 489struct RacyStacks { 490 MD5Hash hash[2]; 491 bool operator==(const RacyStacks &other) const { 492 if (hash[0] == other.hash[0] && hash[1] == other.hash[1]) 493 return true; 494 if (hash[0] == other.hash[1] && hash[1] == other.hash[0]) 495 return true; 496 return false; 497 } 498}; 499 500struct RacyAddress { 501 uptr addr_min; 502 uptr addr_max; 503}; 504 505struct FiredSuppression { 506 ReportType type; 507 uptr pc; 508}; 509 510struct Context { 511 Context(); 512 513 bool initialized; 514 515 SyncTab synctab; 516 517 Mutex report_mtx; 518 int nreported; 519 int nmissed_expected; 520 atomic_uint64_t last_symbolize_time_ns; 521 522 ThreadRegistry *thread_registry; 523 524 Vector<RacyStacks> racy_stacks; 525 Vector<RacyAddress> racy_addresses; 526 Vector<FiredSuppression> fired_suppressions; 527 528 Flags flags; 529 530 u64 stat[StatCnt]; 531 u64 int_alloc_cnt[MBlockTypeCount]; 532 u64 int_alloc_siz[MBlockTypeCount]; 533}; 534 535class ScopedInRtl { 536 public: 537 ScopedInRtl(); 538 ~ScopedInRtl(); 539 private: 540 ThreadState*thr_; 541 int in_rtl_; 542 int errno_; 543}; 544 545class ScopedReport { 546 public: 547 explicit ScopedReport(ReportType typ); 548 ~ScopedReport(); 549 550 void AddStack(const StackTrace *stack); 551 void AddMemoryAccess(uptr addr, Shadow s, const StackTrace *stack, 552 const MutexSet *mset); 553 void AddThread(const ThreadContext *tctx); 554 void AddMutex(const SyncVar *s); 555 void AddLocation(uptr addr, uptr size); 556 void AddSleep(u32 stack_id); 557 558 const ReportDesc *GetReport() const; 559 560 private: 561 Context *ctx_; 562 ReportDesc *rep_; 563 564 void AddMutex(u64 id); 565 566 ScopedReport(const ScopedReport&); 567 void operator = (const ScopedReport&); 568}; 569 570void RestoreStack(int tid, const u64 epoch, StackTrace *stk, MutexSet *mset); 571 572void StatAggregate(u64 *dst, u64 *src); 573void StatOutput(u64 *stat); 574void ALWAYS_INLINE INLINE StatInc(ThreadState *thr, StatType typ, u64 n = 1) { 575 if (kCollectStats) 576 thr->stat[typ] += n; 577} 578void ALWAYS_INLINE INLINE StatSet(ThreadState *thr, StatType typ, u64 n) { 579 if (kCollectStats) 580 thr->stat[typ] = n; 581} 582 583void MapShadow(uptr addr, uptr size); 584void MapThreadTrace(uptr addr, uptr size); 585void DontNeedShadowFor(uptr addr, uptr size); 586void InitializeShadowMemory(); 587void InitializeInterceptors(); 588void InitializeDynamicAnnotations(); 589 590void ReportRace(ThreadState *thr); 591bool OutputReport(Context *ctx, 592 const ScopedReport &srep, 593 const ReportStack *suppress_stack1 = 0, 594 const ReportStack *suppress_stack2 = 0); 595bool IsFiredSuppression(Context *ctx, 596 const ScopedReport &srep, 597 const StackTrace &trace); 598bool IsExpectedReport(uptr addr, uptr size); 599bool FrameIsInternal(const ReportStack *frame); 600ReportStack *SkipTsanInternalFrames(ReportStack *ent); 601 602#if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1 603# define DPrintf Printf 604#else 605# define DPrintf(...) 606#endif 607 608#if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 2 609# define DPrintf2 Printf 610#else 611# define DPrintf2(...) 612#endif 613 614u32 CurrentStackId(ThreadState *thr, uptr pc); 615void PrintCurrentStack(ThreadState *thr, uptr pc); 616void PrintCurrentStackSlow(); // uses libunwind 617 618void Initialize(ThreadState *thr); 619int Finalize(ThreadState *thr); 620 621SyncVar* GetJavaSync(ThreadState *thr, uptr pc, uptr addr, 622 bool write_lock, bool create); 623SyncVar* GetAndRemoveJavaSync(ThreadState *thr, uptr pc, uptr addr); 624 625void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, 626 int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic); 627void MemoryAccessImpl(ThreadState *thr, uptr addr, 628 int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic, 629 u64 *shadow_mem, Shadow cur); 630void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, 631 uptr size, bool is_write); 632void MemoryAccessRangeStep(ThreadState *thr, uptr pc, uptr addr, 633 uptr size, uptr step, bool is_write); 634 635const int kSizeLog1 = 0; 636const int kSizeLog2 = 1; 637const int kSizeLog4 = 2; 638const int kSizeLog8 = 3; 639 640void ALWAYS_INLINE INLINE MemoryRead(ThreadState *thr, uptr pc, 641 uptr addr, int kAccessSizeLog) { 642 MemoryAccess(thr, pc, addr, kAccessSizeLog, false, false); 643} 644 645void ALWAYS_INLINE INLINE MemoryWrite(ThreadState *thr, uptr pc, 646 uptr addr, int kAccessSizeLog) { 647 MemoryAccess(thr, pc, addr, kAccessSizeLog, true, false); 648} 649 650void ALWAYS_INLINE INLINE MemoryReadAtomic(ThreadState *thr, uptr pc, 651 uptr addr, int kAccessSizeLog) { 652 MemoryAccess(thr, pc, addr, kAccessSizeLog, false, true); 653} 654 655void ALWAYS_INLINE INLINE MemoryWriteAtomic(ThreadState *thr, uptr pc, 656 uptr addr, int kAccessSizeLog) { 657 MemoryAccess(thr, pc, addr, kAccessSizeLog, true, true); 658} 659 660void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size); 661void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size); 662void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size); 663void IgnoreCtl(ThreadState *thr, bool write, bool begin); 664 665void FuncEntry(ThreadState *thr, uptr pc); 666void FuncExit(ThreadState *thr); 667 668int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached); 669void ThreadStart(ThreadState *thr, int tid, uptr os_id); 670void ThreadFinish(ThreadState *thr); 671int ThreadTid(ThreadState *thr, uptr pc, uptr uid); 672void ThreadJoin(ThreadState *thr, uptr pc, int tid); 673void ThreadDetach(ThreadState *thr, uptr pc, int tid); 674void ThreadFinalize(ThreadState *thr); 675void ThreadSetName(ThreadState *thr, const char *name); 676int ThreadCount(ThreadState *thr); 677void ProcessPendingSignals(ThreadState *thr); 678 679void MutexCreate(ThreadState *thr, uptr pc, uptr addr, 680 bool rw, bool recursive, bool linker_init); 681void MutexDestroy(ThreadState *thr, uptr pc, uptr addr); 682void MutexLock(ThreadState *thr, uptr pc, uptr addr); 683void MutexUnlock(ThreadState *thr, uptr pc, uptr addr); 684void MutexReadLock(ThreadState *thr, uptr pc, uptr addr); 685void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr); 686void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr); 687 688void Acquire(ThreadState *thr, uptr pc, uptr addr); 689void AcquireGlobal(ThreadState *thr, uptr pc); 690void Release(ThreadState *thr, uptr pc, uptr addr); 691void ReleaseStore(ThreadState *thr, uptr pc, uptr addr); 692void AfterSleep(ThreadState *thr, uptr pc); 693 694// The hacky call uses custom calling convention and an assembly thunk. 695// It is considerably faster that a normal call for the caller 696// if it is not executed (it is intended for slow paths from hot functions). 697// The trick is that the call preserves all registers and the compiler 698// does not treat it as a call. 699// If it does not work for you, use normal call. 700#if TSAN_DEBUG == 0 701// The caller may not create the stack frame for itself at all, 702// so we create a reserve stack frame for it (1024b must be enough). 703#define HACKY_CALL(f) \ 704 __asm__ __volatile__("sub $1024, %%rsp;" \ 705 "/*.cfi_adjust_cfa_offset 1024;*/" \ 706 ".hidden " #f "_thunk;" \ 707 "call " #f "_thunk;" \ 708 "add $1024, %%rsp;" \ 709 "/*.cfi_adjust_cfa_offset -1024;*/" \ 710 ::: "memory", "cc"); 711#else 712#define HACKY_CALL(f) f() 713#endif 714 715void TraceSwitch(ThreadState *thr); 716uptr TraceTopPC(ThreadState *thr); 717uptr TraceSize(); 718uptr TraceParts(); 719Trace *ThreadTrace(int tid); 720 721extern "C" void __tsan_trace_switch(); 722void ALWAYS_INLINE INLINE TraceAddEvent(ThreadState *thr, FastState fs, 723 EventType typ, u64 addr) { 724 DCHECK_GE((int)typ, 0); 725 DCHECK_LE((int)typ, 7); 726 DCHECK_EQ(GetLsb(addr, 61), addr); 727 StatInc(thr, StatEvents); 728 u64 pos = fs.GetTracePos(); 729 if (UNLIKELY((pos % kTracePartSize) == 0)) { 730#ifndef TSAN_GO 731 HACKY_CALL(__tsan_trace_switch); 732#else 733 TraceSwitch(thr); 734#endif 735 } 736 Event *trace = (Event*)GetThreadTrace(fs.tid()); 737 Event *evp = &trace[pos]; 738 Event ev = (u64)addr | ((u64)typ << 61); 739 *evp = ev; 740} 741 742} // namespace __tsan 743 744#endif // TSAN_RTL_H 745