tsan_rtl.cc revision 069ce828e3057819ee34426496ea7080f7cc52f0
17ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany//===-- tsan_rtl.cc ---------------------------------------------*- C++ -*-===// 27ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 37ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// The LLVM Compiler Infrastructure 47ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 57ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// This file is distributed under the University of Illinois Open Source 67ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// License. See LICENSE.TXT for details. 77ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 87ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany//===----------------------------------------------------------------------===// 97ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 107ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// This file is a part of ThreadSanitizer (TSan), a race detector. 117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// Main file (entry points) for the TSan run-time. 137ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany//===----------------------------------------------------------------------===// 147ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 157ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_defs.h" 167ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_platform.h" 177ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_rtl.h" 187ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_interface.h" 197ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_atomic.h" 207ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_mman.h" 217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_placement_new.h" 227ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_suppressions.h" 237ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 247ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvolatile int __tsan_stop = 0; 257ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 267ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyextern "C" void __tsan_resume() { 277ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany __tsan_stop = 0; 287ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 297ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 307ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanynamespace __tsan { 317ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 327ac41484ea322e0ea5774df681660269f5dc321eKostya SerebryanyTHREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGN(64); 337ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystatic char ctx_placeholder[sizeof(Context)] ALIGN(64); 347ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 357ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystatic Context *ctx; 367ac41484ea322e0ea5774df681660269f5dc321eKostya SerebryanyContext *CTX() { 377ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return ctx; 387ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 397ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 407ac41484ea322e0ea5774df681660269f5dc321eKostya SerebryanyContext::Context() 417ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany : initialized() 427ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , report_mtx(MutexTypeReport, StatMtxReport) 437ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , nreported() 447ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , nmissed_expected() 457ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , thread_mtx(MutexTypeThreads, StatMtxThreads) 467ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , racy_stacks(MBlockRacyStacks) 477ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , racy_addresses(MBlockRacyAddresses) { 487ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 497ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 507ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// The objects are allocated in TLS, so one may rely on zero-initialization. 517ac41484ea322e0ea5774df681660269f5dc321eKostya SerebryanyThreadState::ThreadState(Context *ctx, int tid, u64 epoch, 527ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr stk_addr, uptr stk_size, 537ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr tls_addr, uptr tls_size) 547ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany : fast_state(tid, epoch) 557ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Do not touch these, rely on zero initialization, 567ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // they may be accessed before the ctor. 577ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // , fast_ignore_reads() 587ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // , fast_ignore_writes() 597ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // , in_rtl() 607ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , shadow_stack_pos(&shadow_stack[0]) 617ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , tid(tid) 627ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , func_call_count() 637ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , stk_addr(stk_addr) 647ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , stk_size(stk_size) 657ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , tls_addr(tls_addr) 667ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , tls_size(tls_size) { 677ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 687ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 697ac41484ea322e0ea5774df681660269f5dc321eKostya SerebryanyThreadContext::ThreadContext(int tid) 707ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany : tid(tid) 717ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , unique_id() 727ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , user_id() 737ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , thr() 747ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , status(ThreadStatusInvalid) 757ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , detached() 767ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , reuse_count() 777ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , epoch0() 787ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , epoch1() 797ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany , dead_next() { 807ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 817ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 827ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid Initialize(ThreadState *thr) { 837ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Thread safe because done before all threads exist. 847ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany static bool is_initialized = false; 857ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (is_initialized) 867ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 877ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany is_initialized = true; 887ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ScopedInRtl in_rtl; 897ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany InitializeInterceptors(); 907ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany const char *env = InitializePlatform(); 917ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany InitializeMutex(); 927ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany InitializeDynamicAnnotations(); 937ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ctx = new(ctx_placeholder) Context; 947ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany InitializeShadowMemory(); 957ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ctx->dead_list_size = 0; 967ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ctx->dead_list_head = 0; 977ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ctx->dead_list_tail = 0; 987ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany InitializeFlags(&ctx->flags, env); 997ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany InitializeSuppressions(); 1007ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1017ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (ctx->flags.verbosity) 1027ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Printf("***** Running under ThreadSanitizer v2 (pid=%d) *****\n", GetPid()); 1037ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1047ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Initialize thread 0. 1057ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ctx->thread_seq = 0; 1067ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int tid = ThreadCreate(thr, 0, 0, true); 1077ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK_EQ(tid, 0); 1087ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ThreadStart(thr, tid); 1097ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK_EQ(thr->in_rtl, 1); 1107ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ctx->initialized = true; 1117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (__tsan_stop) { 1137ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Printf("ThreadSanitizer is suspended at startup.\n"); 1147ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany while (__tsan_stop); 1157ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1167ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 1177ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1187ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyint Finalize(ThreadState *thr) { 1197ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ScopedInRtl in_rtl; 1207ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Context *ctx = __tsan::ctx; 1217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany bool failed = false; 1227ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1237ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Be very careful beyond that point. 1247ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // All bets are off. Everything is destroyed. 1257ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ThreadFinish(thr); 1267ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ThreadFinalize(thr); 1277ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany FinalizeFlags(&ctx->flags); 1287ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1297ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (ctx->nreported) { 1307ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany failed = true; 1317ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Printf("ThreadSanitizer: reported %d warnings\n", ctx->nreported); 1327ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1337ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1347ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (ctx->nmissed_expected) { 1357ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany failed = true; 1367ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Printf("ThreadSanitizer: missed %d expected races\n", 1377ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ctx->nmissed_expected); 1387ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1397ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1407ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatOutput(ctx->stat); 1417ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany FinalizeSuppressions(); 1427ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany FinalizePlatform(); 1437ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1447ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany const int exitcode = failed ? flags()->exitcode : 0; 1457ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany const int log_fileno = flags()->log_fileno; 1467ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany __tsan::ctx->~Context(); 1477ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany __tsan::ctx = 0; 1487ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1497ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany InternalAllocStatAggregate(ctx, thr); 1507ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1517ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int i = 0; i < (int)MBlockTypeCount; i++) { 1527ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (ctx->int_alloc_cnt[i] == 0 && ctx->int_alloc_siz[i] == 0) 1537ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany continue; 1547ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany InternalScopedBuf<char> tmp(1024); 1557ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Snprintf(tmp, tmp.Size(), "ThreadSanitizer: Internal memory leak: " 1567ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany "type=%d count=%lld size=%lld\n", 1577ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany (int)i, ctx->int_alloc_cnt[i], ctx->int_alloc_siz[i]); 1587ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany internal_write(log_fileno, tmp, internal_strlen(tmp)); 1597ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1607ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1617ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return exitcode; 1627ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 1637ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1647ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystatic void TraceSwitch(ThreadState *thr) { 1657ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ScopedInRtl in_rtl; 1667ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Lock l(&thr->trace.mtx); 1677ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany unsigned trace = (thr->fast_state.epoch() / kTracePartSize) % kTraceParts; 1687ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany TraceHeader *hdr = &thr->trace.headers[trace]; 1697ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany hdr->epoch0 = thr->fast_state.epoch(); 1707ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany hdr->stack0.ObtainCurrent(thr, 0); 1717ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 1727ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1737ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyextern "C" void __tsan_trace_switch() { 1747ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany TraceSwitch(cur_thread()); 1757ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 1767ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1777ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyextern "C" void __tsan_report_race() { 1787ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ReportRace(cur_thread()); 1797ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 1807ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1817ac41484ea322e0ea5774df681660269f5dc321eKostya SerebryanyALWAYS_INLINE 1827ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystatic Shadow LoadShadow(u64 *p) { 1837ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 raw = atomic_load((atomic_uint64_t*)p, memory_order_relaxed); 1847ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return Shadow(raw); 1857ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 1867ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1877ac41484ea322e0ea5774df681660269f5dc321eKostya SerebryanyALWAYS_INLINE 1887ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystatic void StoreShadow(u64 *sp, u64 s) { 1897ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany atomic_store((atomic_uint64_t*)sp, s, memory_order_relaxed); 1907ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 1917ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1927ac41484ea322e0ea5774df681660269f5dc321eKostya SerebryanyALWAYS_INLINE 1937ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystatic void StoreIfNotYetStored(u64 *sp, u64 *s) { 1947ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StoreShadow(sp, *s); 1957ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany *s = 0; 1967ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 1977ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1987ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystatic inline void HandleRace(ThreadState *thr, u64 *shadow_mem, 1997ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Shadow cur, Shadow old) { 2007ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->racy_state[0] = cur.raw(); 2017ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->racy_state[1] = old.raw(); 2027ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->racy_shadow_addr = shadow_mem; 2037ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany HACKY_CALL(__tsan_report_race); 2047ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2057ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2067ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystatic inline bool BothReads(Shadow s, int kAccessIsWrite) { 2077ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return !kAccessIsWrite && !s.is_write(); 2087ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2097ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2107ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystatic inline bool OldIsRWStronger(Shadow old, int kAccessIsWrite) { 2117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return old.is_write() || !kAccessIsWrite; 2127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2137ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2147ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystatic inline bool OldIsRWWeaker(Shadow old, int kAccessIsWrite) { 2157ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return !old.is_write() || kAccessIsWrite; 2167ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2177ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2187ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystatic inline bool OldIsInSameSynchEpoch(Shadow old, ThreadState *thr) { 2197ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return old.epoch() >= thr->fast_synch_epoch; 2207ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2227ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystatic inline bool HappensBefore(Shadow old, ThreadState *thr) { 2237ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return thr->clock.get(old.tid()) >= old.epoch(); 2247ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2257ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2267ac41484ea322e0ea5774df681660269f5dc321eKostya SerebryanyALWAYS_INLINE 2277ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MemoryAccessImpl(ThreadState *thr, uptr addr, 2287ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int kAccessSizeLog, bool kAccessIsWrite, FastState fast_state, 2297ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 *shadow_mem, Shadow cur) { 2307ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatMop); 2317ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead); 2327ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog)); 2337ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2347ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // This potentially can live in an MMX/SSE scratch register. 2357ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // The required intrinsics are: 2367ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // __m128i _mm_move_epi64(__m128i*); 2377ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // _mm_storel_epi64(u64*, __m128i); 2387ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 store_word = cur.raw(); 2397ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2407ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // scan all the shadow values and dispatch to 4 categories: 2417ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // same, replace, candidate and race (see comments below). 2427ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // we consider only 3 cases regarding access sizes: 2437ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // equal, intersect and not intersect. initially I considered 2447ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // larger and smaller as well, it allowed to replace some 2457ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // 'candidates' with 'same' or 'replace', but I think 2467ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // it's just not worth it (performance- and complexity-wise). 2477ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2487ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Shadow old(0); 2497ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (kShadowCnt == 1) { 2507ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int idx = 0; 2517ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 2527ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } else if (kShadowCnt == 2) { 2537ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int idx = 0; 2547ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 2557ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany idx = 1; 2567ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 2577ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } else if (kShadowCnt == 4) { 2587ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int idx = 0; 2597ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 2607ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany idx = 1; 2617ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 2627ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany idx = 2; 2637ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 2647ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany idx = 3; 2657ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 2667ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } else if (kShadowCnt == 8) { 2677ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int idx = 0; 2687ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 2697ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany idx = 1; 2707ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 2717ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany idx = 2; 2727ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 2737ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany idx = 3; 2747ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 2757ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany idx = 4; 2767ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 2777ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany idx = 5; 2787ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 2797ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany idx = 6; 2807ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 2817ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany idx = 7; 2827ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_update_shadow_word_inl.h" 2837ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } else { 2847ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK(false); 2857ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 2867ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2877ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // we did not find any races and had already stored 2887ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // the current access info, so we are done 2897ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (LIKELY(store_word == 0)) 2907ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 2917ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // choose a random candidate slot and replace it 2927ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word); 2937ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatShadowReplace); 2947ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 2957ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany RACE: 2967ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany HandleRace(thr, shadow_mem, cur, old); 2977ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 2987ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2997ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3007ac41484ea322e0ea5774df681660269f5dc321eKostya SerebryanyALWAYS_INLINE 3017ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MemoryAccess(ThreadState *thr, uptr pc, uptr addr, 3027ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int kAccessSizeLog, bool kAccessIsWrite) { 3037ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 *shadow_mem = (u64*)MemToShadow(addr); 3047ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DPrintf2("#%d: tsan::OnMemoryAccess: @%p %p size=%d" 3057ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany " is_write=%d shadow_mem=%p {%llx, %llx, %llx, %llx}\n", 3067ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany (int)thr->fast_state.tid(), (void*)pc, (void*)addr, 3077ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany (int)(1 << kAccessSizeLog), kAccessIsWrite, shadow_mem, 3087ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany shadow_mem[0], shadow_mem[1], shadow_mem[2], shadow_mem[3]); 3097ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#if TSAN_DEBUG 3107ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (!IsAppMem(addr)) { 3117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Printf("Access to non app mem %lx\n", addr); 3127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK(IsAppMem(addr)); 3137ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 3147ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (!IsShadowMem((uptr)shadow_mem)) { 3157ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Printf("Bad shadow addr %p (%lx)\n", shadow_mem, addr); 3167ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK(IsShadowMem((uptr)shadow_mem)); 3177ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 3187ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 3197ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3207ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany FastState fast_state = thr->fast_state; 3217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (fast_state.GetIgnoreBit()) 3227ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 3237ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany fast_state.IncrementEpoch(); 3247ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->fast_state = fast_state; 3257ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Shadow cur(fast_state); 3267ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany cur.SetAddr0AndSizeLog(addr & 7, kAccessSizeLog); 3277ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany cur.SetWrite(kAccessIsWrite); 3287ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3297ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // We must not store to the trace if we do not store to the shadow. 3307ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // That is, this call must be moved somewhere below. 3317ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany TraceAddEvent(thr, fast_state.epoch(), EventTypeMop, pc); 3327ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3337ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany MemoryAccessImpl(thr, addr, kAccessSizeLog, kAccessIsWrite, fast_state, 3347ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany shadow_mem, cur); 3357ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 3367ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3377ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystatic void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size, 3387ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 val) { 3397ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (size == 0) 3407ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 3417ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // FIXME: fix me. 3427ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr offset = addr % kShadowCell; 3437ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (offset) { 3447ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany offset = kShadowCell - offset; 3457ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (size <= offset) 3467ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 3477ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany addr += offset; 3487ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany size -= offset; 3497ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 3507ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK_EQ(addr % 8, 0); 3517ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK(IsAppMem(addr)); 3527ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK(IsAppMem(addr + size - 1)); 3537ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany (void)thr; 3547ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany (void)pc; 3557ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Some programs mmap like hundreds of GBs but actually used a small part. 3567ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // So, it's better to report a false positive on the memory 3577ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // then to hang here senselessly. 3587ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany const uptr kMaxResetSize = 1024*1024*1024; 3597ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (size > kMaxResetSize) 3607ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany size = kMaxResetSize; 3617ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany size = (size + 7) & ~7; 3627ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 *p = (u64*)MemToShadow(addr); 3637ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK(IsShadowMem((uptr)p)); 3647ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK(IsShadowMem((uptr)(p + size * kShadowCnt / kShadowCell - 1))); 3657ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // FIXME: may overwrite a part outside the region 3667ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (uptr i = 0; i < size * kShadowCnt / kShadowCell; i++) 3677ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany p[i] = val; 3687ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 3697ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3707ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size) { 3717ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany MemoryRangeSet(thr, pc, addr, size, 0); 3727ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 3737ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3747ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size) { 3757ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany MemoryAccessRange(thr, pc, addr, size, true); 376069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov Shadow s(thr->fast_state); 377069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov s.MarkAsFreed(); 378069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov s.SetWrite(true); 379069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov s.SetAddr0AndSizeLog(0, 3); 380069ce828e3057819ee34426496ea7080f7cc52f0Dmitry Vyukov MemoryRangeSet(thr, pc, addr, size, s.raw()); 3817ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 3827ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3837ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid FuncEntry(ThreadState *thr, uptr pc) { 3847ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK_EQ(thr->in_rtl, 0); 3857ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatFuncEnter); 3867ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DPrintf2("#%d: tsan::FuncEntry %p\n", (int)thr->fast_state.tid(), (void*)pc); 3877ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->fast_state.IncrementEpoch(); 3887ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany TraceAddEvent(thr, thr->fast_state.epoch(), EventTypeFuncEnter, pc); 3897ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3907ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Shadow stack maintenance can be replaced with 3917ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // stack unwinding during trace switch (which presumably must be faster). 3927ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK(thr->shadow_stack_pos >= &thr->shadow_stack[0]); 3937ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK(thr->shadow_stack_pos < &thr->shadow_stack[kShadowStackSize]); 3947ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->shadow_stack_pos[0] = pc; 3957ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->shadow_stack_pos++; 3967ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 3977ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3987ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid FuncExit(ThreadState *thr) { 3997ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK_EQ(thr->in_rtl, 0); 4007ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatFuncExit); 4017ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DPrintf2("#%d: tsan::FuncExit\n", (int)thr->fast_state.tid()); 4027ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->fast_state.IncrementEpoch(); 4037ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany TraceAddEvent(thr, thr->fast_state.epoch(), EventTypeFuncExit, 0); 4047ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 4057ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK(thr->shadow_stack_pos > &thr->shadow_stack[0]); 4067ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK(thr->shadow_stack_pos < &thr->shadow_stack[kShadowStackSize]); 4077ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->shadow_stack_pos--; 4087ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 4097ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 4107ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid IgnoreCtl(ThreadState *thr, bool write, bool begin) { 4117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DPrintf("#%d: IgnoreCtl(%d, %d)\n", thr->tid, write, begin); 4127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->ignore_reads_and_writes += begin ? 1 : -1; 4137ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK_GE(thr->ignore_reads_and_writes, 0); 4147ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (thr->ignore_reads_and_writes) 4157ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->fast_state.SetIgnoreBit(); 4167ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany else 4177ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->fast_state.ClearIgnoreBit(); 4187ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 4197ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 4207ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid InternalAllocStatAggregate(Context *ctx, ThreadState *thr) { 4217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int i = 0; i < (int)MBlockTypeCount; i++) { 4227ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ctx->int_alloc_cnt[i] += thr->int_alloc_cnt[i]; 4237ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany ctx->int_alloc_siz[i] += thr->int_alloc_siz[i]; 4247ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->int_alloc_cnt[i] = 0; 4257ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->int_alloc_siz[i] = 0; 4267ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 4277ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 4287ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 4297ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#if TSAN_DEBUG 4307ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid build_consistency_debug() {} 4317ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#else 4327ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid build_consistency_release() {} 4337ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 4347ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 4357ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#if TSAN_COLLECT_STATS 4367ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid build_consistency_stats() {} 4377ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#else 4387ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid build_consistency_nostats() {} 4397ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 4407ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 4417ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#if TSAN_SHADOW_COUNT == 1 4427ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid build_consistency_shadow1() {} 4437ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#elif TSAN_SHADOW_COUNT == 2 4447ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid build_consistency_shadow2() {} 4457ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#elif TSAN_SHADOW_COUNT == 4 4467ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid build_consistency_shadow4() {} 4477ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#else 4487ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid build_consistency_shadow8() {} 4497ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 4507ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 4517ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} // namespace __tsan 4527ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 4537ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// Must be included in this file to make sure everything is inlined. 4547ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_interface_inl.h" 455