1603c4be006d8c53905d736bf1f19a49f5ce98276Alexey Samsonov//===-- tsan_mutex.cc -----------------------------------------------------===// 27ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 37ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// The LLVM Compiler Infrastructure 47ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 57ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// This file is distributed under the University of Illinois Open Source 67ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// License. See LICENSE.TXT for details. 77ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 87ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany//===----------------------------------------------------------------------===// 97ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 107ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// This file is a part of ThreadSanitizer (TSan), a race detector. 117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany//===----------------------------------------------------------------------===// 130969bcf2c936126b1f6e636b978aade8fc207437Alexey Samsonov#include "sanitizer_common/sanitizer_libc.h" 147ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_mutex.h" 157ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_platform.h" 167ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_rtl.h" 177ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 187ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanynamespace __tsan { 197ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 207ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// Simple reader-writer spin-mutex. Optimized for not-so-contended case. 217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// Readers have preference, can possibly starvate writers. 227ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 237ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// The table fixes what mutexes can be locked under what mutexes. 247ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// E.g. if the row for MutexTypeThreads contains MutexTypeReport, 257ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// then Report mutex can be locked while under Threads mutex. 267ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// The leaf mutexes can be locked under any other mutexes. 277ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// Recursive locking is not supported. 2886277eb844c4983c81de62d7c050e92fe7155788Stephen Hines#if SANITIZER_DEBUG && !SANITIZER_GO 297ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyconst MutexType MutexTypeLeaf = (MutexType)-1; 307ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystatic MutexType CanLockTab[MutexTypeCount][MutexTypeCount] = { 31f4e4f9393ed1cf9cbefaafc6ea8fd9b89fea4bcfDmitry Vyukov /*0 MutexTypeInvalid*/ {}, 32f4e4f9393ed1cf9cbefaafc6ea8fd9b89fea4bcfDmitry Vyukov /*1 MutexTypeTrace*/ {MutexTypeLeaf}, 33f4e4f9393ed1cf9cbefaafc6ea8fd9b89fea4bcfDmitry Vyukov /*2 MutexTypeThreads*/ {MutexTypeReport}, 346a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines /*3 MutexTypeReport*/ {MutexTypeSyncVar, 3594d8b448e046f3976b13ab79130c389115754022Dmitry Vyukov MutexTypeMBlock, MutexTypeJavaMBlock}, 362d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines /*4 MutexTypeSyncVar*/ {MutexTypeDDetector}, 376a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines /*5 MutexTypeSyncTab*/ {}, // unused 38f4e4f9393ed1cf9cbefaafc6ea8fd9b89fea4bcfDmitry Vyukov /*6 MutexTypeSlab*/ {MutexTypeLeaf}, 39f4e4f9393ed1cf9cbefaafc6ea8fd9b89fea4bcfDmitry Vyukov /*7 MutexTypeAnnotations*/ {}, 406a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines /*8 MutexTypeAtExit*/ {MutexTypeSyncVar}, 41f4e4f9393ed1cf9cbefaafc6ea8fd9b89fea4bcfDmitry Vyukov /*9 MutexTypeMBlock*/ {MutexTypeSyncVar}, 42f4e4f9393ed1cf9cbefaafc6ea8fd9b89fea4bcfDmitry Vyukov /*10 MutexTypeJavaMBlock*/ {MutexTypeSyncVar}, 432d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines /*11 MutexTypeDDetector*/ {}, 447ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany}; 457ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 467ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystatic bool CanLockAdj[MutexTypeCount][MutexTypeCount]; 47d62237995d0fc50697e375ea50f015e996162884Dmitry Vyukov#endif 487ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 497ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid InitializeMutex() { 5086277eb844c4983c81de62d7c050e92fe7155788Stephen Hines#if SANITIZER_DEBUG && !SANITIZER_GO 517ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Build the "can lock" adjacency matrix. 527ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // If [i][j]==true, then one can lock mutex j while under mutex i. 537ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany const int N = MutexTypeCount; 547ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int cnt[N] = {}; 557ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany bool leaf[N] = {}; 567ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int i = 1; i < N; i++) { 577ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int j = 0; j < N; j++) { 582135d8a7f4ba30fe35ed02d5e6ffd59a95b26219Alexey Samsonov MutexType z = CanLockTab[i][j]; 597ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (z == MutexTypeInvalid) 607ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany continue; 617ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (z == MutexTypeLeaf) { 627ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK(!leaf[i]); 637ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany leaf[i] = true; 647ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany continue; 657ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 662135d8a7f4ba30fe35ed02d5e6ffd59a95b26219Alexey Samsonov CHECK(!CanLockAdj[i][(int)z]); 672135d8a7f4ba30fe35ed02d5e6ffd59a95b26219Alexey Samsonov CanLockAdj[i][(int)z] = true; 687ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany cnt[i]++; 697ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 707ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 717ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int i = 0; i < N; i++) { 727ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK(!leaf[i] || cnt[i] == 0); 737ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 747ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Add leaf mutexes. 757ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int i = 0; i < N; i++) { 767ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (!leaf[i]) 777ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany continue; 787ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int j = 0; j < N; j++) { 797ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (i == j || leaf[j] || j == MutexTypeInvalid) 807ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany continue; 817ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK(!CanLockAdj[j][i]); 827ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CanLockAdj[j][i] = true; 837ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 847ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 857ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Build the transitive closure. 867ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany bool CanLockAdj2[MutexTypeCount][MutexTypeCount]; 877ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int i = 0; i < N; i++) { 887ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int j = 0; j < N; j++) { 897ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CanLockAdj2[i][j] = CanLockAdj[i][j]; 907ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 917ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 927ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int k = 0; k < N; k++) { 937ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int i = 0; i < N; i++) { 947ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int j = 0; j < N; j++) { 957ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (CanLockAdj2[i][k] && CanLockAdj2[k][j]) { 967ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CanLockAdj2[i][j] = true; 977ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 987ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 997ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1007ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1017ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#if 0 102b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("Can lock graph:\n"); 1037ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int i = 0; i < N; i++) { 1047ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int j = 0; j < N; j++) { 105b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("%d ", CanLockAdj[i][j]); 1067ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 107b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("\n"); 1087ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 109b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("Can lock graph closure:\n"); 1107ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int i = 0; i < N; i++) { 1117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int j = 0; j < N; j++) { 112b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("%d ", CanLockAdj2[i][j]); 1137ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 114b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("\n"); 1157ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1167ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 1177ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Verify that the graph is acyclic. 1187ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int i = 0; i < N; i++) { 1197ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (CanLockAdj2[i][i]) { 120b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("Mutex %d participates in a cycle\n", i); 1217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Die(); 1227ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1237ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 124d62237995d0fc50697e375ea50f015e996162884Dmitry Vyukov#endif 1257ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 1267ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1272d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesInternalDeadlockDetector::InternalDeadlockDetector() { 1287ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Rely on zero initialization because some mutexes can be locked before ctor. 1297ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 1307ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 13186277eb844c4983c81de62d7c050e92fe7155788Stephen Hines#if SANITIZER_DEBUG && !SANITIZER_GO 1322d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesvoid InternalDeadlockDetector::Lock(MutexType t) { 133b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov // Printf("LOCK %d @%zu\n", t, seq_ + 1); 134ad9da372f962495b3487685232d09390be841b1cDmitry Vyukov CHECK_GT(t, MutexTypeInvalid); 135ad9da372f962495b3487685232d09390be841b1cDmitry Vyukov CHECK_LT(t, MutexTypeCount); 1367ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 max_seq = 0; 1377ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 max_idx = MutexTypeInvalid; 1387ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int i = 0; i != MutexTypeCount; i++) { 1397ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (locked_[i] == 0) 1407ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany continue; 1417ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK_NE(locked_[i], max_seq); 1427ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (max_seq < locked_[i]) { 1437ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany max_seq = locked_[i]; 1447ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany max_idx = i; 1457ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1467ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1477ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany locked_[t] = ++seq_; 1487ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (max_idx == MutexTypeInvalid) 1497ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 150b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov // Printf(" last %d @%zu\n", max_idx, max_seq); 1517ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (!CanLockAdj[max_idx][t]) { 152b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("ThreadSanitizer: internal deadlock detected\n"); 153b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("ThreadSanitizer: can't lock %d while under %zu\n", 154e954101f6602ac181a2c3accfbbad0ae51b0bf7cAlexey Samsonov t, (uptr)max_idx); 155ff35f1d82b4f145b3477ef27a7a2e7b63c486988Dmitry Vyukov CHECK(0); 1567ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1577ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 1587ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1592d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesvoid InternalDeadlockDetector::Unlock(MutexType t) { 160b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov // Printf("UNLO %d @%zu #%zu\n", t, seq_, locked_[t]); 1617ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK(locked_[t]); 1627ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany locked_[t] = 0; 1637ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 1646a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines 1656a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hinesvoid InternalDeadlockDetector::CheckNoLocks() { 1666a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines for (int i = 0; i != MutexTypeCount; i++) { 1676a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines CHECK_EQ(locked_[i], 0); 1686a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines } 1696a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines} 1706a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines#endif 1716a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines 1726a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hinesvoid CheckNoLocks(ThreadState *thr) { 17386277eb844c4983c81de62d7c050e92fe7155788Stephen Hines#if SANITIZER_DEBUG && !SANITIZER_GO 1746a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines thr->internal_deadlock_detector.CheckNoLocks(); 175d62237995d0fc50697e375ea50f015e996162884Dmitry Vyukov#endif 1766a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines} 1777ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1787ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyconst uptr kUnlocked = 0; 1797ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyconst uptr kWriteLock = 1; 1807ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyconst uptr kReadLock = 2; 1817ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1827ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyclass Backoff { 1837ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany public: 1847ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Backoff() 1857ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany : iter_() { 1867ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1877ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1887ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany bool Do() { 1897ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (iter_++ < kActiveSpinIters) 1907ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany proc_yield(kActiveSpinCnt); 1917ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany else 1920969bcf2c936126b1f6e636b978aade8fc207437Alexey Samsonov internal_sched_yield(); 1937ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return true; 1947ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1957ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1967ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 Contention() const { 1977ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 active = iter_ % kActiveSpinIters; 1987ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 passive = iter_ - active; 1997ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return active + 10 * passive; 2007ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 2017ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2027ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany private: 2037ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int iter_; 2047ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany static const int kActiveSpinIters = 10; 2057ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany static const int kActiveSpinCnt = 20; 2067ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany}; 2077ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2087ac41484ea322e0ea5774df681660269f5dc321eKostya SerebryanyMutex::Mutex(MutexType type, StatType stat_type) { 2097ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK_GT(type, MutexTypeInvalid); 2107ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK_LT(type, MutexTypeCount); 21186277eb844c4983c81de62d7c050e92fe7155788Stephen Hines#if SANITIZER_DEBUG 2127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany type_ = type; 2137ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 2147ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#if TSAN_COLLECT_STATS 2157ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany stat_type_ = stat_type; 2167ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 2177ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany atomic_store(&state_, kUnlocked, memory_order_relaxed); 2187ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2197ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2207ac41484ea322e0ea5774df681660269f5dc321eKostya SerebryanyMutex::~Mutex() { 2217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK_EQ(atomic_load(&state_, memory_order_relaxed), kUnlocked); 2227ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2237ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2247ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid Mutex::Lock() { 22586277eb844c4983c81de62d7c050e92fe7155788Stephen Hines#if SANITIZER_DEBUG && !SANITIZER_GO 2262d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines cur_thread()->internal_deadlock_detector.Lock(type_); 2277ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 2287ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr cmp = kUnlocked; 2297ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (atomic_compare_exchange_strong(&state_, &cmp, kWriteLock, 2307ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany memory_order_acquire)) 2317ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 2327ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (Backoff backoff; backoff.Do();) { 2337ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (atomic_load(&state_, memory_order_relaxed) == kUnlocked) { 2347ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany cmp = kUnlocked; 2357ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (atomic_compare_exchange_weak(&state_, &cmp, kWriteLock, 2367ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany memory_order_acquire)) { 23786277eb844c4983c81de62d7c050e92fe7155788Stephen Hines#if TSAN_COLLECT_STATS && !SANITIZER_GO 2387ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(cur_thread(), stat_type_, backoff.Contention()); 2397ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 2407ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 2417ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 2427ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 2437ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 2447ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2457ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2467ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid Mutex::Unlock() { 2477ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr prev = atomic_fetch_sub(&state_, kWriteLock, memory_order_release); 2487ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany (void)prev; 2497ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK_NE(prev & kWriteLock, 0); 25086277eb844c4983c81de62d7c050e92fe7155788Stephen Hines#if SANITIZER_DEBUG && !SANITIZER_GO 2512d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines cur_thread()->internal_deadlock_detector.Unlock(type_); 2527ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 2537ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2547ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2557ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid Mutex::ReadLock() { 25686277eb844c4983c81de62d7c050e92fe7155788Stephen Hines#if SANITIZER_DEBUG && !SANITIZER_GO 2572d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines cur_thread()->internal_deadlock_detector.Lock(type_); 2587ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 2597ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr prev = atomic_fetch_add(&state_, kReadLock, memory_order_acquire); 2607ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if ((prev & kWriteLock) == 0) 2617ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 2627ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (Backoff backoff; backoff.Do();) { 2637ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany prev = atomic_load(&state_, memory_order_acquire); 2647ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if ((prev & kWriteLock) == 0) { 26586277eb844c4983c81de62d7c050e92fe7155788Stephen Hines#if TSAN_COLLECT_STATS && !SANITIZER_GO 2667ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(cur_thread(), stat_type_, backoff.Contention()); 2677ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 2687ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 2697ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 2707ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 2717ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2727ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2737ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid Mutex::ReadUnlock() { 2747ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr prev = atomic_fetch_sub(&state_, kReadLock, memory_order_release); 2757ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany (void)prev; 2767ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK_EQ(prev & kWriteLock, 0); 2777ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK_GT(prev & ~kWriteLock, 0); 27886277eb844c4983c81de62d7c050e92fe7155788Stephen Hines#if SANITIZER_DEBUG && !SANITIZER_GO 2792d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines cur_thread()->internal_deadlock_detector.Unlock(type_); 2807ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 2817ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2827ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 283ff35f1d82b4f145b3477ef27a7a2e7b63c486988Dmitry Vyukovvoid Mutex::CheckLocked() { 284ff35f1d82b4f145b3477ef27a7a2e7b63c486988Dmitry Vyukov CHECK_NE(atomic_load(&state_, memory_order_relaxed), 0); 285ff35f1d82b4f145b3477ef27a7a2e7b63c486988Dmitry Vyukov} 286ff35f1d82b4f145b3477ef27a7a2e7b63c486988Dmitry Vyukov 2877ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} // namespace __tsan 288