1603c4be006d8c53905d736bf1f19a49f5ce98276Alexey Samsonov//===-- tsan_rtl_mutex.cc -------------------------------------------------===// 27ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 37ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// The LLVM Compiler Infrastructure 47ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 57ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// This file is distributed under the University of Illinois Open Source 67ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// License. See LICENSE.TXT for details. 77ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 87ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany//===----------------------------------------------------------------------===// 97ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 107ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// This file is a part of ThreadSanitizer (TSan), a race detector. 117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany//===----------------------------------------------------------------------===// 137ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 142d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#include <sanitizer_common/sanitizer_deadlock_detector_interface.h> 152d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#include <sanitizer_common/sanitizer_stackdepot.h> 162d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 177ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_rtl.h" 18fd513907acd3e2eaa02070edaa24df66648dc2acDmitry Vyukov#include "tsan_flags.h" 197ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_sync.h" 207ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_report.h" 217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_symbolize.h" 229adce675d5978dc8c584bafe724208bbab4ae72aDmitry Vyukov#include "tsan_platform.h" 237ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 247ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanynamespace __tsan { 257ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 262d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesvoid ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r); 272d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 282d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstruct Callback : DDCallback { 292d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ThreadState *thr; 302d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines uptr pc; 312d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 322d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines Callback(ThreadState *thr, uptr pc) 332d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines : thr(thr) 342d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines , pc(pc) { 352d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines DDCallback::pt = thr->dd_pt; 362d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines DDCallback::lt = thr->dd_lt; 372d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 382d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 392d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines virtual u32 Unwind() { 402d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return CurrentStackId(thr, pc); 412d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 422d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines virtual int UniqueTid() { 432d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return thr->unique_id; 442d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 452d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines}; 462d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 472d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesvoid DDMutexInit(ThreadState *thr, uptr pc, SyncVar *s) { 482d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines Callback cb(thr, pc); 492d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ctx->dd->MutexInit(&cb, &s->dd); 502d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines s->dd.ctx = s->GetId(); 512d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 522d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 532d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstatic void ReportMutexMisuse(ThreadState *thr, uptr pc, ReportType typ, 542d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines uptr addr, u64 mid) { 555d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // In Go, these misuses are either impossible, or detected by std lib, 565d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // or false positives (e.g. unlock in a different thread). 575d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (kGoMode) 585d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines return; 592d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ThreadRegistryLock l(ctx->thread_registry); 602d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ScopedReport rep(typ); 612d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines rep.AddMutex(mid); 622d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines StackTrace trace; 632d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines trace.ObtainCurrent(thr, pc); 645d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines rep.AddStack(&trace, true); 652d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines rep.AddLocation(addr, 1); 665d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines OutputReport(thr, rep); 672d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 682d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 697ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MutexCreate(ThreadState *thr, uptr pc, uptr addr, 70c20e9ba760a93706c9160133cdcce62194f9694fDmitry Vyukov bool rw, bool recursive, bool linker_init) { 71e954101f6602ac181a2c3accfbbad0ae51b0bf7cAlexey Samsonov DPrintf("#%d: MutexCreate %zx\n", thr->tid, addr); 727ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatMutexCreate); 733285866e45a8521c56ba6209daf3c9f91f844fd3Dmitry Vyukov if (!linker_init && IsAppMem(addr)) { 743285866e45a8521c56ba6209daf3c9f91f844fd3Dmitry Vyukov CHECK(!thr->is_freeing); 753285866e45a8521c56ba6209daf3c9f91f844fd3Dmitry Vyukov thr->is_freeing = true; 76334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov MemoryWrite(thr, pc, addr, kSizeLog1); 773285866e45a8521c56ba6209daf3c9f91f844fd3Dmitry Vyukov thr->is_freeing = false; 783285866e45a8521c56ba6209daf3c9f91f844fd3Dmitry Vyukov } 795d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true); 807ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany s->is_rw = rw; 817ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany s->is_recursive = recursive; 822e933fc077595b18de37d2ed44e8f14c6053a432Dmitry Vyukov s->is_linker_init = linker_init; 835d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (kCppMode && s->creation_stack_id == 0) 845d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines s->creation_stack_id = CurrentStackId(thr, pc); 857ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany s->mtx.Unlock(); 867ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 877ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 887ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MutexDestroy(ThreadState *thr, uptr pc, uptr addr) { 89e954101f6602ac181a2c3accfbbad0ae51b0bf7cAlexey Samsonov DPrintf("#%d: MutexDestroy %zx\n", thr->tid, addr); 907ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatMutexDestroy); 910c2feef1067818db0ede4531a2e71c9b5595d57aDmitry Vyukov#ifndef TSAN_GO 920c2feef1067818db0ede4531a2e71c9b5595d57aDmitry Vyukov // Global mutexes not marked as LINKER_INITIALIZED 930c2feef1067818db0ede4531a2e71c9b5595d57aDmitry Vyukov // cause tons of not interesting reports, so just ignore it. 940c2feef1067818db0ede4531a2e71c9b5595d57aDmitry Vyukov if (IsGlobalVar(addr)) 950c2feef1067818db0ede4531a2e71c9b5595d57aDmitry Vyukov return; 960c2feef1067818db0ede4531a2e71c9b5595d57aDmitry Vyukov#endif 973285866e45a8521c56ba6209daf3c9f91f844fd3Dmitry Vyukov if (IsAppMem(addr)) { 983285866e45a8521c56ba6209daf3c9f91f844fd3Dmitry Vyukov CHECK(!thr->is_freeing); 993285866e45a8521c56ba6209daf3c9f91f844fd3Dmitry Vyukov thr->is_freeing = true; 100334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov MemoryWrite(thr, pc, addr, kSizeLog1); 1013285866e45a8521c56ba6209daf3c9f91f844fd3Dmitry Vyukov thr->is_freeing = false; 1023285866e45a8521c56ba6209daf3c9f91f844fd3Dmitry Vyukov } 1035d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr); 1045d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (s == 0) 1055d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines return; 1065d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (flags()->detect_deadlocks) { 1075d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines Callback cb(thr, pc); 1085d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines ctx->dd->MutexDestroy(&cb, &s->dd); 1095d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines ctx->dd->MutexInit(&cb, &s->dd); 1105d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines } 1115d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines bool unlock_locked = false; 11255e711ed818d7553d62eba4b3a2fad3e2723f697Dmitry Vyukov if (flags()->report_destroy_locked 11355e711ed818d7553d62eba4b3a2fad3e2723f697Dmitry Vyukov && s->owner_tid != SyncVar::kInvalidTid 11455e711ed818d7553d62eba4b3a2fad3e2723f697Dmitry Vyukov && !s->is_broken) { 11555e711ed818d7553d62eba4b3a2fad3e2723f697Dmitry Vyukov s->is_broken = true; 1165d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines unlock_locked = true; 1175d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines } 1185d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines u64 mid = s->GetId(); 1195d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines u32 last_lock = s->last_lock; 1205d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (!unlock_locked) 1215d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines s->Reset(); // must not reset it before the report is printed 1225d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines s->mtx.Unlock(); 1235d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (unlock_locked) { 1242bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov ThreadRegistryLock l(ctx->thread_registry); 12555e711ed818d7553d62eba4b3a2fad3e2723f697Dmitry Vyukov ScopedReport rep(ReportTypeMutexDestroyLocked); 1265d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines rep.AddMutex(mid); 12755e711ed818d7553d62eba4b3a2fad3e2723f697Dmitry Vyukov StackTrace trace; 12855e711ed818d7553d62eba4b3a2fad3e2723f697Dmitry Vyukov trace.ObtainCurrent(thr, pc); 12955e711ed818d7553d62eba4b3a2fad3e2723f697Dmitry Vyukov rep.AddStack(&trace); 1305d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines FastState last(last_lock); 131ad9da372f962495b3487685232d09390be841b1cDmitry Vyukov RestoreStack(last.tid(), last.epoch(), &trace, 0); 1325d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines rep.AddStack(&trace, true); 1335d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines rep.AddLocation(addr, 1); 1345d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines OutputReport(thr, rep); 1355d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines } 1365d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (unlock_locked) { 1375d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr); 1385d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (s != 0) { 1395d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines s->Reset(); 1405d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines s->mtx.Unlock(); 1415d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines } 1427ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1435d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines thr->mset.Remove(mid); 1445d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines // s will be destroyed and freed in MetaMap::FreeBlock. 1457ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 1467ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1472d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesvoid MutexLock(ThreadState *thr, uptr pc, uptr addr, int rec, bool try_lock) { 1488354faeec647dc5b420726fa61616189392d5045Dmitry Vyukov DPrintf("#%d: MutexLock %zx rec=%d\n", thr->tid, addr, rec); 1498354faeec647dc5b420726fa61616189392d5045Dmitry Vyukov CHECK_GT(rec, 0); 1509adce675d5978dc8c584bafe724208bbab4ae72aDmitry Vyukov if (IsAppMem(addr)) 151334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov MemoryReadAtomic(thr, pc, addr, kSizeLog1); 1525d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true); 1537ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->fast_state.IncrementEpoch(); 154ad9da372f962495b3487685232d09390be841b1cDmitry Vyukov TraceAddEvent(thr, thr->fast_state, EventTypeLock, s->GetId()); 1552d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines bool report_double_lock = false; 1567ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (s->owner_tid == SyncVar::kInvalidTid) { 1577ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK_EQ(s->recursion, 0); 1587ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany s->owner_tid = thr->tid; 159332c62b52b3603be872b28bd3ea5e739aa28cd05Dmitry Vyukov s->last_lock = thr->fast_state.raw(); 1607ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } else if (s->owner_tid == thr->tid) { 1617ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK_GT(s->recursion, 0); 1622d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } else if (flags()->report_mutex_bugs && !s->is_broken) { 1632d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines s->is_broken = true; 1642d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines report_double_lock = true; 1657ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1667ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (s->recursion == 0) { 1677ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatMutexLock); 168e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov AcquireImpl(thr, pc, &s->clock); 169e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov AcquireImpl(thr, pc, &s->read_clock); 1707ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } else if (!s->is_recursive) { 1717ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatMutexRecLock); 1727ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1738354faeec647dc5b420726fa61616189392d5045Dmitry Vyukov s->recursion += rec; 174ad9da372f962495b3487685232d09390be841b1cDmitry Vyukov thr->mset.Add(s->GetId(), true, thr->fast_state.epoch()); 1755d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (flags()->detect_deadlocks && (s->recursion - rec) == 0) { 1762d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines Callback cb(thr, pc); 1772d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (!try_lock) 1782d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ctx->dd->MutexBeforeLock(&cb, &s->dd, true); 1792d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ctx->dd->MutexAfterLock(&cb, &s->dd, true, try_lock); 1802d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 1812d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines u64 mid = s->GetId(); 1827ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany s->mtx.Unlock(); 1832d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // Can't touch s after this point. 1842d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (report_double_lock) 1852d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ReportMutexMisuse(thr, pc, ReportTypeMutexDoubleLock, addr, mid); 1862d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (flags()->detect_deadlocks) { 1872d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines Callback cb(thr, pc); 1882d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb)); 1892d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 1907ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 1917ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1928354faeec647dc5b420726fa61616189392d5045Dmitry Vyukovint MutexUnlock(ThreadState *thr, uptr pc, uptr addr, bool all) { 1938354faeec647dc5b420726fa61616189392d5045Dmitry Vyukov DPrintf("#%d: MutexUnlock %zx all=%d\n", thr->tid, addr, all); 1949adce675d5978dc8c584bafe724208bbab4ae72aDmitry Vyukov if (IsAppMem(addr)) 195334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov MemoryReadAtomic(thr, pc, addr, kSizeLog1); 1965d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true); 1977ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->fast_state.IncrementEpoch(); 198ad9da372f962495b3487685232d09390be841b1cDmitry Vyukov TraceAddEvent(thr, thr->fast_state, EventTypeUnlock, s->GetId()); 1998354faeec647dc5b420726fa61616189392d5045Dmitry Vyukov int rec = 0; 2002d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines bool report_bad_unlock = false; 2015d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (kCppMode && (s->recursion == 0 || s->owner_tid != thr->tid)) { 2022d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (flags()->report_mutex_bugs && !s->is_broken) { 2037ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany s->is_broken = true; 2042d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines report_bad_unlock = true; 2057ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 2067ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } else { 2078354faeec647dc5b420726fa61616189392d5045Dmitry Vyukov rec = all ? s->recursion : 1; 2088354faeec647dc5b420726fa61616189392d5045Dmitry Vyukov s->recursion -= rec; 2097ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (s->recursion == 0) { 2107ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatMutexUnlock); 2117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany s->owner_tid = SyncVar::kInvalidTid; 212117699dc17e35e84ea1179428a245ff6b60aa98fDmitry Vyukov ReleaseStoreImpl(thr, pc, &s->clock); 2137ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } else { 2147ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatMutexRecUnlock); 2157ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 2167ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 217ad9da372f962495b3487685232d09390be841b1cDmitry Vyukov thr->mset.Del(s->GetId(), true); 2185d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (flags()->detect_deadlocks && s->recursion == 0 && !report_bad_unlock) { 2192d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines Callback cb(thr, pc); 2202d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ctx->dd->MutexBeforeUnlock(&cb, &s->dd, true); 2212d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 2222d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines u64 mid = s->GetId(); 2237ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany s->mtx.Unlock(); 2242d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // Can't touch s after this point. 2252d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (report_bad_unlock) 2262d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr, mid); 2275d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines if (flags()->detect_deadlocks && !report_bad_unlock) { 2282d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines Callback cb(thr, pc); 2292d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb)); 2302d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 2318354faeec647dc5b420726fa61616189392d5045Dmitry Vyukov return rec; 2327ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2337ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2342d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesvoid MutexReadLock(ThreadState *thr, uptr pc, uptr addr, bool trylock) { 235e954101f6602ac181a2c3accfbbad0ae51b0bf7cAlexey Samsonov DPrintf("#%d: MutexReadLock %zx\n", thr->tid, addr); 2367ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatMutexReadLock); 2379adce675d5978dc8c584bafe724208bbab4ae72aDmitry Vyukov if (IsAppMem(addr)) 238334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov MemoryReadAtomic(thr, pc, addr, kSizeLog1); 2395d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, false); 2407ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->fast_state.IncrementEpoch(); 241ad9da372f962495b3487685232d09390be841b1cDmitry Vyukov TraceAddEvent(thr, thr->fast_state, EventTypeRLock, s->GetId()); 2422d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines bool report_bad_lock = false; 2431da1056127d1dbcacdd035eb4149257848f7c4dfDmitry Vyukov if (s->owner_tid != SyncVar::kInvalidTid) { 2442d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (flags()->report_mutex_bugs && !s->is_broken) { 2452d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines s->is_broken = true; 2462d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines report_bad_lock = true; 2472d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 2481da1056127d1dbcacdd035eb4149257848f7c4dfDmitry Vyukov } 249e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov AcquireImpl(thr, pc, &s->clock); 250332c62b52b3603be872b28bd3ea5e739aa28cd05Dmitry Vyukov s->last_lock = thr->fast_state.raw(); 251ad9da372f962495b3487685232d09390be841b1cDmitry Vyukov thr->mset.Add(s->GetId(), false, thr->fast_state.epoch()); 2522d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (flags()->detect_deadlocks && s->recursion == 0) { 2532d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines Callback cb(thr, pc); 2542d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (!trylock) 2552d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ctx->dd->MutexBeforeLock(&cb, &s->dd, false); 2562d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ctx->dd->MutexAfterLock(&cb, &s->dd, false, trylock); 2572d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 2582d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines u64 mid = s->GetId(); 2597ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany s->mtx.ReadUnlock(); 2602d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // Can't touch s after this point. 2612d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (report_bad_lock) 2622d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadLock, addr, mid); 2632d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (flags()->detect_deadlocks) { 2642d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines Callback cb(thr, pc); 2652d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb)); 2662d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 2677ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2687ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2697ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr) { 270e954101f6602ac181a2c3accfbbad0ae51b0bf7cAlexey Samsonov DPrintf("#%d: MutexReadUnlock %zx\n", thr->tid, addr); 2717ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatMutexReadUnlock); 2729adce675d5978dc8c584bafe724208bbab4ae72aDmitry Vyukov if (IsAppMem(addr)) 273334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov MemoryReadAtomic(thr, pc, addr, kSizeLog1); 2745d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true); 2757ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->fast_state.IncrementEpoch(); 276ad9da372f962495b3487685232d09390be841b1cDmitry Vyukov TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId()); 2772d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines bool report_bad_unlock = false; 2781da1056127d1dbcacdd035eb4149257848f7c4dfDmitry Vyukov if (s->owner_tid != SyncVar::kInvalidTid) { 2792d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (flags()->report_mutex_bugs && !s->is_broken) { 2802d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines s->is_broken = true; 2812d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines report_bad_unlock = true; 2822d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 2831da1056127d1dbcacdd035eb4149257848f7c4dfDmitry Vyukov } 284e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov ReleaseImpl(thr, pc, &s->read_clock); 2852d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (flags()->detect_deadlocks && s->recursion == 0) { 2862d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines Callback cb(thr, pc); 2872d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ctx->dd->MutexBeforeUnlock(&cb, &s->dd, false); 2882d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 2892d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines u64 mid = s->GetId(); 2907ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany s->mtx.Unlock(); 2912d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // Can't touch s after this point. 2922d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines thr->mset.Del(mid, false); 2932d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (report_bad_unlock) 2942d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadUnlock, addr, mid); 2952d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (flags()->detect_deadlocks) { 2962d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines Callback cb(thr, pc); 2972d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb)); 2982d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 2997ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 3007ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3017ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) { 302e954101f6602ac181a2c3accfbbad0ae51b0bf7cAlexey Samsonov DPrintf("#%d: MutexReadOrWriteUnlock %zx\n", thr->tid, addr); 3039adce675d5978dc8c584bafe724208bbab4ae72aDmitry Vyukov if (IsAppMem(addr)) 304334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov MemoryReadAtomic(thr, pc, addr, kSizeLog1); 3055d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true); 306ad9da372f962495b3487685232d09390be841b1cDmitry Vyukov bool write = true; 3072d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines bool report_bad_unlock = false; 3087ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (s->owner_tid == SyncVar::kInvalidTid) { 3097ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Seems to be read unlock. 310ad9da372f962495b3487685232d09390be841b1cDmitry Vyukov write = false; 3117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatMutexReadUnlock); 3127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany thr->fast_state.IncrementEpoch(); 313ad9da372f962495b3487685232d09390be841b1cDmitry Vyukov TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId()); 314e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov ReleaseImpl(thr, pc, &s->read_clock); 3157ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } else if (s->owner_tid == thr->tid) { 3167ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Seems to be write unlock. 317ad9da372f962495b3487685232d09390be841b1cDmitry Vyukov thr->fast_state.IncrementEpoch(); 318ad9da372f962495b3487685232d09390be841b1cDmitry Vyukov TraceAddEvent(thr, thr->fast_state, EventTypeUnlock, s->GetId()); 3197ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK_GT(s->recursion, 0); 3207ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany s->recursion--; 3217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (s->recursion == 0) { 3227ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatMutexUnlock); 3237ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany s->owner_tid = SyncVar::kInvalidTid; 324e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov ReleaseImpl(thr, pc, &s->clock); 3257ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } else { 3267ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatMutexRecUnlock); 3277ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 3287ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } else if (!s->is_broken) { 3297ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany s->is_broken = true; 3302d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines report_bad_unlock = true; 3317ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 332ad9da372f962495b3487685232d09390be841b1cDmitry Vyukov thr->mset.Del(s->GetId(), write); 3332d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (flags()->detect_deadlocks && s->recursion == 0) { 3342d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines Callback cb(thr, pc); 3352d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ctx->dd->MutexBeforeUnlock(&cb, &s->dd, write); 3362d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 3372d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines u64 mid = s->GetId(); 3387ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany s->mtx.Unlock(); 3392d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // Can't touch s after this point. 3402d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (report_bad_unlock) 3412d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr, mid); 3422d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (flags()->detect_deadlocks) { 3432d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines Callback cb(thr, pc); 3442d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb)); 3452d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 3467ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 3477ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 34811f5309ec1bf13430c8a3a16f177d9e8e1190e38Dmitry Vyukovvoid MutexRepair(ThreadState *thr, uptr pc, uptr addr) { 34911f5309ec1bf13430c8a3a16f177d9e8e1190e38Dmitry Vyukov DPrintf("#%d: MutexRepair %zx\n", thr->tid, addr); 3505d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true); 35111f5309ec1bf13430c8a3a16f177d9e8e1190e38Dmitry Vyukov s->owner_tid = SyncVar::kInvalidTid; 35211f5309ec1bf13430c8a3a16f177d9e8e1190e38Dmitry Vyukov s->recursion = 0; 35311f5309ec1bf13430c8a3a16f177d9e8e1190e38Dmitry Vyukov s->mtx.Unlock(); 35411f5309ec1bf13430c8a3a16f177d9e8e1190e38Dmitry Vyukov} 35511f5309ec1bf13430c8a3a16f177d9e8e1190e38Dmitry Vyukov 3567ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid Acquire(ThreadState *thr, uptr pc, uptr addr) { 357e954101f6602ac181a2c3accfbbad0ae51b0bf7cAlexey Samsonov DPrintf("#%d: Acquire %zx\n", thr->tid, addr); 358e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov if (thr->ignore_sync) 359e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov return; 3605d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, false); 361e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov AcquireImpl(thr, pc, &s->clock); 3627ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany s->mtx.ReadUnlock(); 3637ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 3647ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3652bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonovstatic void UpdateClockCallback(ThreadContextBase *tctx_base, void *arg) { 3662bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov ThreadState *thr = reinterpret_cast<ThreadState*>(arg); 3672bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base); 3682bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov if (tctx->status == ThreadStatusRunning) 3692bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov thr->clock.set(tctx->tid, tctx->thr->fast_state.epoch()); 3702bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov else 3712bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov thr->clock.set(tctx->tid, tctx->epoch1); 3722bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov} 3732bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov 374538f1ba8cb57cfa02d25f8f922feb00975e0a286Dmitry Vyukovvoid AcquireGlobal(ThreadState *thr, uptr pc) { 375e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov DPrintf("#%d: AcquireGlobal\n", thr->tid); 376e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov if (thr->ignore_sync) 377e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov return; 3782d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ThreadRegistryLock l(ctx->thread_registry); 3792d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ctx->thread_registry->RunCallbackForEachThreadLocked( 3802bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov UpdateClockCallback, thr); 381538f1ba8cb57cfa02d25f8f922feb00975e0a286Dmitry Vyukov} 382538f1ba8cb57cfa02d25f8f922feb00975e0a286Dmitry Vyukov 3837ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid Release(ThreadState *thr, uptr pc, uptr addr) { 384e954101f6602ac181a2c3accfbbad0ae51b0bf7cAlexey Samsonov DPrintf("#%d: Release %zx\n", thr->tid, addr); 385e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov if (thr->ignore_sync) 386e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov return; 3875d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true); 388e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov thr->fast_state.IncrementEpoch(); 389e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov // Can't increment epoch w/o writing to the trace as well. 390e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); 391e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov ReleaseImpl(thr, pc, &s->clock); 3927ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany s->mtx.Unlock(); 3937ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 3947ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3959d150bdb433ddd092073dabd87ba15aa176603a1Dmitry Vyukovvoid ReleaseStore(ThreadState *thr, uptr pc, uptr addr) { 3969d150bdb433ddd092073dabd87ba15aa176603a1Dmitry Vyukov DPrintf("#%d: ReleaseStore %zx\n", thr->tid, addr); 397e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov if (thr->ignore_sync) 398e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov return; 3995d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true); 400e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov thr->fast_state.IncrementEpoch(); 401e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov // Can't increment epoch w/o writing to the trace as well. 402e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); 403e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov ReleaseStoreImpl(thr, pc, &s->clock); 4049d150bdb433ddd092073dabd87ba15aa176603a1Dmitry Vyukov s->mtx.Unlock(); 4059d150bdb433ddd092073dabd87ba15aa176603a1Dmitry Vyukov} 4069d150bdb433ddd092073dabd87ba15aa176603a1Dmitry Vyukov 407848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukov#ifndef TSAN_GO 4082bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonovstatic void UpdateSleepClockCallback(ThreadContextBase *tctx_base, void *arg) { 4092bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov ThreadState *thr = reinterpret_cast<ThreadState*>(arg); 4102bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base); 4112bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov if (tctx->status == ThreadStatusRunning) 4122bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov thr->last_sleep_clock.set(tctx->tid, tctx->thr->fast_state.epoch()); 4132bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov else 4142bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov thr->last_sleep_clock.set(tctx->tid, tctx->epoch1); 4152bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov} 4162bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov 417848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukovvoid AfterSleep(ThreadState *thr, uptr pc) { 418e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov DPrintf("#%d: AfterSleep %zx\n", thr->tid); 419e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov if (thr->ignore_sync) 420e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov return; 421848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukov thr->last_sleep_stack_id = CurrentStackId(thr, pc); 4222d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ThreadRegistryLock l(ctx->thread_registry); 4232d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ctx->thread_registry->RunCallbackForEachThreadLocked( 4242bbd8bec77c2fdb41c5f5b6cb0d83d22bc576650Alexey Samsonov UpdateSleepClockCallback, thr); 425848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukov} 426848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukov#endif 427848531192777acecf79747dc7c1ffeedf5c1da9fDmitry Vyukov 428e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukovvoid AcquireImpl(ThreadState *thr, uptr pc, SyncClock *c) { 429e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov if (thr->ignore_sync) 430e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov return; 4312d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines thr->clock.set(thr->fast_state.epoch()); 432e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov thr->clock.acquire(c); 433e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov StatInc(thr, StatSyncAcquire); 434e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov} 435e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov 436e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukovvoid ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) { 437e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov if (thr->ignore_sync) 438e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov return; 4392d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines thr->clock.set(thr->fast_state.epoch()); 440e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov thr->fast_synch_epoch = thr->fast_state.epoch(); 441e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov thr->clock.release(c); 442e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov StatInc(thr, StatSyncRelease); 443e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov} 444e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov 445e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukovvoid ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c) { 446e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov if (thr->ignore_sync) 447e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov return; 4482d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines thr->clock.set(thr->fast_state.epoch()); 449e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov thr->fast_synch_epoch = thr->fast_state.epoch(); 450e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov thr->clock.ReleaseStore(c); 451e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov StatInc(thr, StatSyncRelease); 452e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov} 453e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov 454e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukovvoid AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) { 455e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov if (thr->ignore_sync) 456e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov return; 4572d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines thr->clock.set(thr->fast_state.epoch()); 458e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov thr->fast_synch_epoch = thr->fast_state.epoch(); 459e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov thr->clock.acq_rel(c); 460e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov StatInc(thr, StatSyncAcquire); 461e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov StatInc(thr, StatSyncRelease); 462e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov} 463e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov 4642d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesvoid ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r) { 4652d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (r == 0) 4662d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return; 4672d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ThreadRegistryLock l(ctx->thread_registry); 4682d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ScopedReport rep(ReportTypeDeadlock); 4692d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines for (int i = 0; i < r->n; i++) { 4702d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines rep.AddMutex(r->loop[i].mtx_ctx0); 4712d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines rep.AddUniqueTid((int)r->loop[i].thr_ctx); 4722d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines rep.AddThread((int)r->loop[i].thr_ctx); 4732d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 4745d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines InternalScopedBuffer<StackTrace> stacks(2 * DDReport::kMaxLoopSize); 4752d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines uptr dummy_pc = 0x42; 4762d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines for (int i = 0; i < r->n; i++) { 4772d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines uptr size; 4782d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines for (int j = 0; j < (flags()->second_deadlock_stack ? 2 : 1); j++) { 4792d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines u32 stk = r->loop[i].stk[j]; 4802d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (stk) { 4812d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines const uptr *trace = StackDepotGet(stk, &size); 4822d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines stacks[i].Init(const_cast<uptr *>(trace), size); 4832d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } else { 4842d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // Sometimes we fail to extract the stack trace (FIXME: investigate), 4852d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // but we should still produce some stack trace in the report. 4862d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines stacks[i].Init(&dummy_pc, 1); 4872d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 4885d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines rep.AddStack(&stacks[i], true); 4892d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 4902d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 4915d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines OutputReport(thr, rep); 4922d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 4932d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 4947ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} // namespace __tsan 495