1603c4be006d8c53905d736bf1f19a49f5ce98276Alexey Samsonov//===-- tsan_rtl_thread.cc ------------------------------------------------===// 27ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 37ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// The LLVM Compiler Infrastructure 47ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 57ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// This file is distributed under the University of Illinois Open Source 67ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// License. See LICENSE.TXT for details. 77ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 87ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany//===----------------------------------------------------------------------===// 97ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 107ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// This file is a part of ThreadSanitizer (TSan), a race detector. 117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany//===----------------------------------------------------------------------===// 137ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1447b1634df012507799eb39aa17d4022d748ba67bAlexey Samsonov#include "sanitizer_common/sanitizer_placement_new.h" 157ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_rtl.h" 167ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_mman.h" 177ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_platform.h" 187ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_report.h" 197ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_sync.h" 207ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanynamespace __tsan { 227ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 232bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov// ThreadContext implementation. 242bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov 252bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey SamsonovThreadContext::ThreadContext(int tid) 262bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov : ThreadContextBase(tid) 272bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov , thr() 282bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov , sync() 292bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov , epoch0() 309743d74426ae43898e4da55e591b09be18f8aa6eDmitry Vyukov , epoch1() { 312bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov} 322bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov 336af642eaf764434ac6f28f242e7f081156bce9e3Dmitry Vyukov#ifndef TSAN_GO 346af642eaf764434ac6f28f242e7f081156bce9e3Dmitry VyukovThreadContext::~ThreadContext() { 356af642eaf764434ac6f28f242e7f081156bce9e3Dmitry Vyukov} 366af642eaf764434ac6f28f242e7f081156bce9e3Dmitry Vyukov#endif 376af642eaf764434ac6f28f242e7f081156bce9e3Dmitry Vyukov 382bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonovvoid ThreadContext::OnDead() { 392bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov sync.Reset(); 402bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov} 412bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov 422bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonovvoid ThreadContext::OnJoined(void *arg) { 432bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov ThreadState *caller_thr = static_cast<ThreadState *>(arg); 44e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov AcquireImpl(caller_thr, 0, &sync); 45b186c1905d471243cf2981ca754397bc16976392Dmitry Vyukov sync.Reset(); 462bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov} 472bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov 482bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonovstruct OnCreatedArgs { 492bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov ThreadState *thr; 502bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov uptr pc; 512bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov}; 522bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov 532bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonovvoid ThreadContext::OnCreated(void *arg) { 542bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov thr = 0; 55491852ecf05d927cf543292ded98dcc545799b4dDmitry Vyukov if (tid == 0) 56491852ecf05d927cf543292ded98dcc545799b4dDmitry Vyukov return; 57491852ecf05d927cf543292ded98dcc545799b4dDmitry Vyukov OnCreatedArgs *args = static_cast<OnCreatedArgs *>(arg); 58491852ecf05d927cf543292ded98dcc545799b4dDmitry Vyukov args->thr->fast_state.IncrementEpoch(); 59491852ecf05d927cf543292ded98dcc545799b4dDmitry Vyukov // Can't increment epoch w/o writing to the trace as well. 60491852ecf05d927cf543292ded98dcc545799b4dDmitry Vyukov TraceAddEvent(args->thr, args->thr->fast_state, EventTypeMop, 0); 61e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov ReleaseImpl(args->thr, 0, &sync); 622c5284e0f87e101e177a151fae5f557bcf6f664cDmitry Vyukov creation_stack_id = CurrentStackId(args->thr, args->pc); 63491852ecf05d927cf543292ded98dcc545799b4dDmitry Vyukov if (reuse_count == 0) 64491852ecf05d927cf543292ded98dcc545799b4dDmitry Vyukov StatInc(args->thr, StatThreadMaxTid); 652bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov} 662bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov 67ce85e03620f64ce7e4cd0598f2e93090f52a9e99Dmitry Vyukovvoid ThreadContext::OnReset() { 682bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov sync.Reset(); 699743d74426ae43898e4da55e591b09be18f8aa6eDmitry Vyukov FlushUnneededShadowMemory(GetThreadTrace(tid), TraceSize() * sizeof(Event)); 709743d74426ae43898e4da55e591b09be18f8aa6eDmitry Vyukov //!!! FlushUnneededShadowMemory(GetThreadTraceHeader(tid), sizeof(Trace)); 712bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov} 722bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov 732bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonovstruct OnStartedArgs { 742bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov ThreadState *thr; 752bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov uptr stk_addr; 762bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov uptr stk_size; 772bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov uptr tls_addr; 782bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov uptr tls_size; 792bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov}; 802bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov 812bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonovvoid ThreadContext::OnStarted(void *arg) { 822bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov OnStartedArgs *args = static_cast<OnStartedArgs*>(arg); 83491852ecf05d927cf543292ded98dcc545799b4dDmitry Vyukov thr = args->thr; 842bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov // RoundUp so that one trace part does not contain events 852bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov // from different threads. 862bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov epoch0 = RoundUp(epoch1 + 1, kTracePartSize); 872bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov epoch1 = (u64)-1; 882d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines new(thr) ThreadState(ctx, tid, unique_id, epoch0, reuse_count, 892d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines args->stk_addr, args->stk_size, args->tls_addr, args->tls_size); 9001a7ce809bf7cc627d73c045c70bcca9891f632cDmitry Vyukov#ifndef TSAN_GO 9101a7ce809bf7cc627d73c045c70bcca9891f632cDmitry Vyukov thr->shadow_stack = &ThreadTrace(thr->tid)->shadow_stack[0]; 9201a7ce809bf7cc627d73c045c70bcca9891f632cDmitry Vyukov thr->shadow_stack_pos = thr->shadow_stack; 9301a7ce809bf7cc627d73c045c70bcca9891f632cDmitry Vyukov thr->shadow_stack_end = thr->shadow_stack + kShadowStackSize; 9401a7ce809bf7cc627d73c045c70bcca9891f632cDmitry Vyukov#else 952bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov // Setup dynamic shadow stack. 962bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov const int kInitStackSize = 8; 9701a7ce809bf7cc627d73c045c70bcca9891f632cDmitry Vyukov thr->shadow_stack = (uptr*)internal_alloc(MBlockShadowStack, 982bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov kInitStackSize * sizeof(uptr)); 9901a7ce809bf7cc627d73c045c70bcca9891f632cDmitry Vyukov thr->shadow_stack_pos = thr->shadow_stack; 10001a7ce809bf7cc627d73c045c70bcca9891f632cDmitry Vyukov thr->shadow_stack_end = thr->shadow_stack + kInitStackSize; 1012bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov#endif 102e716b602e8066ede297b5e0be4d23720c26cdcf9Dmitry Vyukov#ifndef TSAN_GO 10301a7ce809bf7cc627d73c045c70bcca9891f632cDmitry Vyukov AllocatorThreadStart(thr); 104e716b602e8066ede297b5e0be4d23720c26cdcf9Dmitry Vyukov#endif 1052d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (flags()->detect_deadlocks) { 1062d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines thr->dd_pt = ctx->dd->CreatePhysicalThread(); 1072d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines thr->dd_lt = ctx->dd->CreateLogicalThread(unique_id); 1082d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 1092bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov thr->fast_synch_epoch = epoch0; 110e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov AcquireImpl(thr, 0, &sync); 1112bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov thr->fast_state.SetHistorySize(flags()->history_size); 1122bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov const uptr trace = (epoch0 / kTracePartSize) % TraceParts(); 1139743d74426ae43898e4da55e591b09be18f8aa6eDmitry Vyukov Trace *thr_trace = ThreadTrace(thr->tid); 1149743d74426ae43898e4da55e591b09be18f8aa6eDmitry Vyukov thr_trace->headers[trace].epoch0 = epoch0; 1152bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov StatInc(thr, StatSyncAcquire); 1169743d74426ae43898e4da55e591b09be18f8aa6eDmitry Vyukov sync.Reset(); 1172bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov DPrintf("#%d: ThreadStart epoch=%zu stk_addr=%zx stk_size=%zx " 1182bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov "tls_addr=%zx tls_size=%zx\n", 119036f9336f7cc7e755287a317f71a7bec7534a7f7Alexey Samsonov tid, (uptr)epoch0, args->stk_addr, args->stk_size, 120036f9336f7cc7e755287a317f71a7bec7534a7f7Alexey Samsonov args->tls_addr, args->tls_size); 1212bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov thr->is_alive = true; 1222bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov} 1237ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1242bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonovvoid ThreadContext::OnFinished() { 1252bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov if (!detached) { 1262bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov thr->fast_state.IncrementEpoch(); 1272bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov // Can't increment epoch w/o writing to the trace as well. 1282bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); 129e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov ReleaseImpl(thr, 0, &sync); 1302bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov } 1312bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov epoch1 = thr->fast_state.epoch(); 1322bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov 1332d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (flags()->detect_deadlocks) { 1342d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ctx->dd->DestroyPhysicalThread(thr->dd_pt); 1352d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ctx->dd->DestroyLogicalThread(thr->dd_lt); 1362d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 1375d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines ctx->metamap.OnThreadIdle(thr); 1382bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov#ifndef TSAN_GO 1392bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov AllocatorThreadFinish(thr); 1402bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov#endif 1412bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov thr->~ThreadState(); 1422d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines StatAggregate(ctx->stat, thr->stat); 1432bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov thr = 0; 1442bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov} 1452bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov 1464536cb1fa7734133f404acb413589d7a6d314f4aDmitry Vyukov#ifndef TSAN_GO 1474536cb1fa7734133f404acb413589d7a6d314f4aDmitry Vyukovstruct ThreadLeak { 1484536cb1fa7734133f404acb413589d7a6d314f4aDmitry Vyukov ThreadContext *tctx; 1494536cb1fa7734133f404acb413589d7a6d314f4aDmitry Vyukov int count; 1504536cb1fa7734133f404acb413589d7a6d314f4aDmitry Vyukov}; 1514536cb1fa7734133f404acb413589d7a6d314f4aDmitry Vyukov 1524536cb1fa7734133f404acb413589d7a6d314f4aDmitry Vyukovstatic void MaybeReportThreadLeak(ThreadContextBase *tctx_base, void *arg) { 1534536cb1fa7734133f404acb413589d7a6d314f4aDmitry Vyukov Vector<ThreadLeak> &leaks = *(Vector<ThreadLeak>*)arg; 1542bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base); 1554536cb1fa7734133f404acb413589d7a6d314f4aDmitry Vyukov if (tctx->detached || tctx->status != ThreadStatusFinished) 1567ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 1574536cb1fa7734133f404acb413589d7a6d314f4aDmitry Vyukov for (uptr i = 0; i < leaks.Size(); i++) { 1584536cb1fa7734133f404acb413589d7a6d314f4aDmitry Vyukov if (leaks[i].tctx->creation_stack_id == tctx->creation_stack_id) { 1594536cb1fa7734133f404acb413589d7a6d314f4aDmitry Vyukov leaks[i].count++; 1604536cb1fa7734133f404acb413589d7a6d314f4aDmitry Vyukov return; 1614536cb1fa7734133f404acb413589d7a6d314f4aDmitry Vyukov } 1624536cb1fa7734133f404acb413589d7a6d314f4aDmitry Vyukov } 1634536cb1fa7734133f404acb413589d7a6d314f4aDmitry Vyukov ThreadLeak leak = {tctx, 1}; 1644536cb1fa7734133f404acb413589d7a6d314f4aDmitry Vyukov leaks.PushBack(leak); 1657ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 1664536cb1fa7734133f404acb413589d7a6d314f4aDmitry Vyukov#endif 1677ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1682d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#ifndef TSAN_GO 1692d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstatic void ReportIgnoresEnabled(ThreadContext *tctx, IgnoreSet *set) { 1702d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (tctx->tid == 0) { 1712d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines Printf("ThreadSanitizer: main thread finished with ignores enabled\n"); 1722d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } else { 1732d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines Printf("ThreadSanitizer: thread T%d %s finished with ignores enabled," 1742d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines " created at:\n", tctx->tid, tctx->name); 1752d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines PrintStack(SymbolizeStackId(tctx->creation_stack_id)); 176dc563c0efbba5aec49b20a4d74e020feb75d7e8aDmitry Vyukov } 1772d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines Printf(" One of the following ignores was not ended" 1782d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines " (in order of probability)\n"); 1792d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines for (uptr i = 0; i < set->Size(); i++) { 1802d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines Printf(" Ignore was enabled at:\n"); 1812d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines PrintStack(SymbolizeStackId(set->At(i))); 182e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov } 1832d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines Die(); 184dc563c0efbba5aec49b20a4d74e020feb75d7e8aDmitry Vyukov} 185dc563c0efbba5aec49b20a4d74e020feb75d7e8aDmitry Vyukov 1862d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstatic void ThreadCheckIgnore(ThreadState *thr) { 1872d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (ctx->after_multithreaded_fork) 1882d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return; 1892d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (thr->ignore_reads_and_writes) 1902d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ReportIgnoresEnabled(thr->tctx, &thr->mop_ignore_set); 1912d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (thr->ignore_sync) 1922d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ReportIgnoresEnabled(thr->tctx, &thr->sync_ignore_set); 1932d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 1942d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#else 1952d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstatic void ThreadCheckIgnore(ThreadState *thr) {} 1962d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#endif 1972d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 1987ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid ThreadFinalize(ThreadState *thr) { 199dc563c0efbba5aec49b20a4d74e020feb75d7e8aDmitry Vyukov ThreadCheckIgnore(thr); 2004536cb1fa7734133f404acb413589d7a6d314f4aDmitry Vyukov#ifndef TSAN_GO 2017ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (!flags()->report_thread_leaks) 2027ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 2032d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ThreadRegistryLock l(ctx->thread_registry); 2044536cb1fa7734133f404acb413589d7a6d314f4aDmitry Vyukov Vector<ThreadLeak> leaks(MBlockScopedBuf); 2052d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ctx->thread_registry->RunCallbackForEachThreadLocked( 2064536cb1fa7734133f404acb413589d7a6d314f4aDmitry Vyukov MaybeReportThreadLeak, &leaks); 2074536cb1fa7734133f404acb413589d7a6d314f4aDmitry Vyukov for (uptr i = 0; i < leaks.Size(); i++) { 2084536cb1fa7734133f404acb413589d7a6d314f4aDmitry Vyukov ScopedReport rep(ReportTypeThreadLeak); 2095d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines rep.AddThread(leaks[i].tctx, true); 2104536cb1fa7734133f404acb413589d7a6d314f4aDmitry Vyukov rep.SetCount(leaks[i].count); 2115d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines OutputReport(thr, rep); 2124536cb1fa7734133f404acb413589d7a6d314f4aDmitry Vyukov } 2134536cb1fa7734133f404acb413589d7a6d314f4aDmitry Vyukov#endif 2147ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2157ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 21654e0a9a391616bbd2a4f0fd2d01076291274cb98Dmitry Vyukovint ThreadCount(ThreadState *thr) { 2172bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov uptr result; 2182bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov ctx->thread_registry->GetNumberOfThreads(0, 0, &result); 2192bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov return (int)result; 2207ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2227ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyint ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached) { 2237ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatThreadCreate); 2242bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov OnCreatedArgs args = { thr, pc }; 2252bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov int tid = ctx->thread_registry->CreateThread(uid, detached, thr->tid, &args); 226e954101f6602ac181a2c3accfbbad0ae51b0bf7cAlexey Samsonov DPrintf("#%d: ThreadCreate tid=%d uid=%zu\n", thr->tid, tid, uid); 2272bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov StatSet(thr, StatThreadMaxAlive, ctx->thread_registry->GetMaxAliveThreads()); 2287ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return tid; 2297ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2307ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 231e0023f74ea88efee329f68391b70f8adc6b21617Dmitry Vyukovvoid ThreadStart(ThreadState *thr, int tid, uptr os_id) { 2327ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr stk_addr = 0; 2337ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr stk_size = 0; 2347ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr tls_addr = 0; 2357ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr tls_size = 0; 236789b6c5669547b550034fc9888059b17e2ff1417Dmitry Vyukov GetThreadStackAndTls(tid == 0, &stk_addr, &stk_size, &tls_addr, &tls_size); 2377ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 238af154b815bbd793874b4cd05be98d571b6579272Dmitry Vyukov if (tid) { 2397ac33ac529ff93a57419f5ddf71b1fde68428577Dmitry Vyukov if (stk_addr && stk_size) 24074172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov MemoryRangeImitateWrite(thr, /*pc=*/ 1, stk_addr, stk_size); 2417ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 242b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov if (tls_addr && tls_size) { 243b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov // Check that the thr object is in tls; 244b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov const uptr thr_beg = (uptr)thr; 245b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov const uptr thr_end = (uptr)thr + sizeof(*thr); 246b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov CHECK_GE(thr_beg, tls_addr); 247b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov CHECK_LE(thr_beg, tls_addr + tls_size); 248b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov CHECK_GE(thr_end, tls_addr); 249b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov CHECK_LE(thr_end, tls_addr + tls_size); 250b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov // Since the thr object is huge, skip it. 25174172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov MemoryRangeImitateWrite(thr, /*pc=*/ 2, tls_addr, thr_beg - tls_addr); 25274172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov MemoryRangeImitateWrite(thr, /*pc=*/ 2, 25374172de3ce25bf52d9d3918c94216f2eef5956e2Dmitry Vyukov thr_end, tls_addr + tls_size - thr_end); 254b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov } 255af154b815bbd793874b4cd05be98d571b6579272Dmitry Vyukov } 2567ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2572d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ThreadRegistry *tr = ctx->thread_registry; 2582bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov OnStartedArgs args = { thr, stk_addr, stk_size, tls_addr, tls_size }; 2592d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines tr->StartThread(tid, os_id, &args); 2602d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 2612d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines tr->Lock(); 2622d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines thr->tctx = (ThreadContext*)tr->GetThreadLocked(tid); 2632d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines tr->Unlock(); 2642d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 2652d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#ifndef TSAN_GO 2662d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (ctx->after_multithreaded_fork) { 2672d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines thr->ignore_interceptors++; 2682d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ThreadIgnoreBegin(thr, 0); 2692d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ThreadIgnoreSyncBegin(thr, 0); 2702d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 2712d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#endif 2727ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2737ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2747ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid ThreadFinish(ThreadState *thr) { 275dc563c0efbba5aec49b20a4d74e020feb75d7e8aDmitry Vyukov ThreadCheckIgnore(thr); 2767ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatThreadFinish); 2777ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (thr->stk_addr && thr->stk_size) 2787ac33ac529ff93a57419f5ddf71b1fde68428577Dmitry Vyukov DontNeedShadowFor(thr->stk_addr, thr->stk_size); 2797ac33ac529ff93a57419f5ddf71b1fde68428577Dmitry Vyukov if (thr->tls_addr && thr->tls_size) 2807ac33ac529ff93a57419f5ddf71b1fde68428577Dmitry Vyukov DontNeedShadowFor(thr->tls_addr, thr->tls_size); 2811fc03d50a87302689482efa24b045a393097b6c3Dmitry Vyukov thr->is_alive = false; 2822bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov ctx->thread_registry->FinishThread(thr->tid); 2832bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov} 2847ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2852bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonovstatic bool FindThreadByUid(ThreadContextBase *tctx, void *arg) { 2862bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov uptr uid = (uptr)arg; 2872bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov if (tctx->user_id == uid && tctx->status != ThreadStatusInvalid) { 2882bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov tctx->user_id = 0; 2892bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov return true; 2907ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 2912bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov return false; 2927ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2937ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2947ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyint ThreadTid(ThreadState *thr, uptr pc, uptr uid) { 2952bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov int res = ctx->thread_registry->FindThread(FindThreadByUid, (void*)uid); 296e954101f6602ac181a2c3accfbbad0ae51b0bf7cAlexey Samsonov DPrintf("#%d: ThreadTid uid=%zu tid=%d\n", thr->tid, uid, res); 297411b2c9d6787b64939fc15fdeec65e9d65ba1a51Dmitry Vyukov return res; 2987ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2997ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3007ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid ThreadJoin(ThreadState *thr, uptr pc, int tid) { 3017ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK_GT(tid, 0); 3027ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK_LT(tid, kMaxTid); 3037ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DPrintf("#%d: ThreadJoin tid=%d\n", thr->tid, tid); 3042bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov ctx->thread_registry->JoinThread(tid, thr); 3057ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 3067ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3077ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid ThreadDetach(ThreadState *thr, uptr pc, int tid) { 3087ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK_GT(tid, 0); 3097ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK_LT(tid, kMaxTid); 3102bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov ctx->thread_registry->DetachThread(tid); 3117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 3127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 313aecf2e5756c6a0de7c146bef67a6e338c7017d55Dmitry Vyukovvoid ThreadSetName(ThreadState *thr, const char *name) { 3142d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ctx->thread_registry->SetThreadName(thr->tid, name); 315aecf2e5756c6a0de7c146bef67a6e338c7017d55Dmitry Vyukov} 316aecf2e5756c6a0de7c146bef67a6e338c7017d55Dmitry Vyukov 3177ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, 3187ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr size, bool is_write) { 3197ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (size == 0) 3207ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 3217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3227ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 *shadow_mem = (u64*)MemToShadow(addr); 3237ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DPrintf2("#%d: MemoryAccessRange: @%p %p size=%d is_write=%d\n", 3247ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->tid, (void*)pc, (void*)addr, 3257ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany (int)size, is_write); 3267ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3277ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#if TSAN_DEBUG 3287ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (!IsAppMem(addr)) { 329b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("Access to non app mem %zx\n", addr); 3307ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK(IsAppMem(addr)); 3317ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 3327ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (!IsAppMem(addr + size - 1)) { 333b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("Access to non app mem %zx\n", addr + size - 1); 3347ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK(IsAppMem(addr + size - 1)); 3357ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 3367ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (!IsShadowMem((uptr)shadow_mem)) { 337b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr); 3387ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK(IsShadowMem((uptr)shadow_mem)); 3397ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 3407ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (!IsShadowMem((uptr)(shadow_mem + size * kShadowCnt / 8 - 1))) { 341b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("Bad shadow addr %p (%zx)\n", 342e954101f6602ac181a2c3accfbbad0ae51b0bf7cAlexey Samsonov shadow_mem + size * kShadowCnt / 8 - 1, addr + size - 1); 3437ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK(IsShadowMem((uptr)(shadow_mem + size * kShadowCnt / 8 - 1))); 3447ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 3457ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 3467ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3477ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatMopRange); 3487ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 34982dbc5195ceedba0e1a9aab92d436614cc4b7ff9Dmitry Vyukov if (*shadow_mem == kShadowRodata) { 35082dbc5195ceedba0e1a9aab92d436614cc4b7ff9Dmitry Vyukov // Access to .rodata section, no races here. 35182dbc5195ceedba0e1a9aab92d436614cc4b7ff9Dmitry Vyukov // Measurements show that it can be 10-20% of all memory accesses. 35282dbc5195ceedba0e1a9aab92d436614cc4b7ff9Dmitry Vyukov StatInc(thr, StatMopRangeRodata); 35382dbc5195ceedba0e1a9aab92d436614cc4b7ff9Dmitry Vyukov return; 35482dbc5195ceedba0e1a9aab92d436614cc4b7ff9Dmitry Vyukov } 35582dbc5195ceedba0e1a9aab92d436614cc4b7ff9Dmitry Vyukov 3567ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany FastState fast_state = thr->fast_state; 3577ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (fast_state.GetIgnoreBit()) 3587ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 3597ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3607ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany fast_state.IncrementEpoch(); 3617ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->fast_state = fast_state; 362385542a2e83a4f37de4232d6c72097c1b7d6d44bDmitry Vyukov TraceAddEvent(thr, fast_state, EventTypeMop, pc); 3637ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3647ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany bool unaligned = (addr % kShadowCell) != 0; 3657ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3667ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Handle unaligned beginning, if any. 3677ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (; addr % kShadowCell && size; addr++, size--) { 3687ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int const kAccessSizeLog = 0; 3697ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Shadow cur(fast_state); 3707ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany cur.SetWrite(is_write); 3717ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany cur.SetAddr0AndSizeLog(addr & (kShadowCell - 1), kAccessSizeLog); 372334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false, 3737ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany shadow_mem, cur); 3747ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 3757ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (unaligned) 3767ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany shadow_mem += kShadowCnt; 3777ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Handle middle part, if any. 3787ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (; size >= kShadowCell; addr += kShadowCell, size -= kShadowCell) { 3797ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int const kAccessSizeLog = 3; 3807ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Shadow cur(fast_state); 3817ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany cur.SetWrite(is_write); 3827ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany cur.SetAddr0AndSizeLog(0, kAccessSizeLog); 383334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false, 3847ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany shadow_mem, cur); 3857ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany shadow_mem += kShadowCnt; 3867ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 3877ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Handle ending, if any. 3887ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (; size; addr++, size--) { 3897ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int const kAccessSizeLog = 0; 3907ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Shadow cur(fast_state); 3917ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany cur.SetWrite(is_write); 3927ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany cur.SetAddr0AndSizeLog(addr & (kShadowCell - 1), kAccessSizeLog); 393334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false, 3947ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany shadow_mem, cur); 3957ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 3967ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 397eaa0190fc9a510a621d871becce0536950c5468dDmitry Vyukov 3987ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} // namespace __tsan 399