1603c4be006d8c53905d736bf1f19a49f5ce98276Alexey Samsonov//===-- tsan_mutex.cc -----------------------------------------------------===// 27ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 37ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// The LLVM Compiler Infrastructure 47ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 57ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// This file is distributed under the University of Illinois Open Source 67ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// License. See LICENSE.TXT for details. 77ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 87ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany//===----------------------------------------------------------------------===// 97ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 107ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// This file is a part of ThreadSanitizer (TSan), a race detector. 117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany//===----------------------------------------------------------------------===// 130969bcf2c936126b1f6e636b978aade8fc207437Alexey Samsonov#include "sanitizer_common/sanitizer_libc.h" 147ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_mutex.h" 157ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_platform.h" 167ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_rtl.h" 177ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 187ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanynamespace __tsan { 197ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 207ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// Simple reader-writer spin-mutex. Optimized for not-so-contended case. 217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// Readers have preference, can possibly starvate writers. 227ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 237ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// The table fixes what mutexes can be locked under what mutexes. 247ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// E.g. if the row for MutexTypeThreads contains MutexTypeReport, 257ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// then Report mutex can be locked while under Threads mutex. 267ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// The leaf mutexes can be locked under any other mutexes. 277ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// Recursive locking is not supported. 28d62237995d0fc50697e375ea50f015e996162884Dmitry Vyukov#if TSAN_DEBUG && !TSAN_GO 297ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyconst MutexType MutexTypeLeaf = (MutexType)-1; 307ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystatic MutexType CanLockTab[MutexTypeCount][MutexTypeCount] = { 31f4e4f9393ed1cf9cbefaafc6ea8fd9b89fea4bcfDmitry Vyukov /*0 MutexTypeInvalid*/ {}, 32f4e4f9393ed1cf9cbefaafc6ea8fd9b89fea4bcfDmitry Vyukov /*1 MutexTypeTrace*/ {MutexTypeLeaf}, 33f4e4f9393ed1cf9cbefaafc6ea8fd9b89fea4bcfDmitry Vyukov /*2 MutexTypeThreads*/ {MutexTypeReport}, 34f4e4f9393ed1cf9cbefaafc6ea8fd9b89fea4bcfDmitry Vyukov /*3 MutexTypeReport*/ {MutexTypeSyncTab, MutexTypeMBlock, 35f4e4f9393ed1cf9cbefaafc6ea8fd9b89fea4bcfDmitry Vyukov MutexTypeJavaMBlock}, 36f4e4f9393ed1cf9cbefaafc6ea8fd9b89fea4bcfDmitry Vyukov /*4 MutexTypeSyncVar*/ {}, 37f4e4f9393ed1cf9cbefaafc6ea8fd9b89fea4bcfDmitry Vyukov /*5 MutexTypeSyncTab*/ {MutexTypeSyncVar}, 38f4e4f9393ed1cf9cbefaafc6ea8fd9b89fea4bcfDmitry Vyukov /*6 MutexTypeSlab*/ {MutexTypeLeaf}, 39f4e4f9393ed1cf9cbefaafc6ea8fd9b89fea4bcfDmitry Vyukov /*7 MutexTypeAnnotations*/ {}, 40f4e4f9393ed1cf9cbefaafc6ea8fd9b89fea4bcfDmitry Vyukov /*8 MutexTypeAtExit*/ {MutexTypeSyncTab}, 41f4e4f9393ed1cf9cbefaafc6ea8fd9b89fea4bcfDmitry Vyukov /*9 MutexTypeMBlock*/ {MutexTypeSyncVar}, 42f4e4f9393ed1cf9cbefaafc6ea8fd9b89fea4bcfDmitry Vyukov /*10 MutexTypeJavaMBlock*/ {MutexTypeSyncVar}, 437ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany}; 447ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 457ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystatic bool CanLockAdj[MutexTypeCount][MutexTypeCount]; 46d62237995d0fc50697e375ea50f015e996162884Dmitry Vyukov#endif 477ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 487ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid InitializeMutex() { 49d62237995d0fc50697e375ea50f015e996162884Dmitry Vyukov#if TSAN_DEBUG && !TSAN_GO 507ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Build the "can lock" adjacency matrix. 517ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // If [i][j]==true, then one can lock mutex j while under mutex i. 527ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany const int N = MutexTypeCount; 537ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int cnt[N] = {}; 547ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany bool leaf[N] = {}; 557ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int i = 1; i < N; i++) { 567ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int j = 0; j < N; j++) { 572135d8a7f4ba30fe35ed02d5e6ffd59a95b26219Alexey Samsonov MutexType z = CanLockTab[i][j]; 587ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (z == MutexTypeInvalid) 597ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany continue; 607ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (z == MutexTypeLeaf) { 617ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK(!leaf[i]); 627ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany leaf[i] = true; 637ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany continue; 647ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 652135d8a7f4ba30fe35ed02d5e6ffd59a95b26219Alexey Samsonov CHECK(!CanLockAdj[i][(int)z]); 662135d8a7f4ba30fe35ed02d5e6ffd59a95b26219Alexey Samsonov CanLockAdj[i][(int)z] = true; 677ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany cnt[i]++; 687ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 697ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 707ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int i = 0; i < N; i++) { 717ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK(!leaf[i] || cnt[i] == 0); 727ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 737ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Add leaf mutexes. 747ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int i = 0; i < N; i++) { 757ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (!leaf[i]) 767ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany continue; 777ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int j = 0; j < N; j++) { 787ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (i == j || leaf[j] || j == MutexTypeInvalid) 797ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany continue; 807ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK(!CanLockAdj[j][i]); 817ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CanLockAdj[j][i] = true; 827ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 837ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 847ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Build the transitive closure. 857ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany bool CanLockAdj2[MutexTypeCount][MutexTypeCount]; 867ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int i = 0; i < N; i++) { 877ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int j = 0; j < N; j++) { 887ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CanLockAdj2[i][j] = CanLockAdj[i][j]; 897ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 907ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 917ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int k = 0; k < N; k++) { 927ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int i = 0; i < N; i++) { 937ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int j = 0; j < N; j++) { 947ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (CanLockAdj2[i][k] && CanLockAdj2[k][j]) { 957ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CanLockAdj2[i][j] = true; 967ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 977ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 987ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 997ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1007ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#if 0 101b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("Can lock graph:\n"); 1027ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int i = 0; i < N; i++) { 1037ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int j = 0; j < N; j++) { 104b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("%d ", CanLockAdj[i][j]); 1057ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 106b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("\n"); 1077ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 108b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("Can lock graph closure:\n"); 1097ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int i = 0; i < N; i++) { 1107ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int j = 0; j < N; j++) { 111b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("%d ", CanLockAdj2[i][j]); 1127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 113b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("\n"); 1147ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1157ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 1167ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Verify that the graph is acyclic. 1177ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int i = 0; i < N; i++) { 1187ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (CanLockAdj2[i][i]) { 119b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("Mutex %d participates in a cycle\n", i); 1207ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Die(); 1217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1227ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 123d62237995d0fc50697e375ea50f015e996162884Dmitry Vyukov#endif 1247ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 1257ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1267ac41484ea322e0ea5774df681660269f5dc321eKostya SerebryanyDeadlockDetector::DeadlockDetector() { 1277ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Rely on zero initialization because some mutexes can be locked before ctor. 1287ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 1297ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 130d62237995d0fc50697e375ea50f015e996162884Dmitry Vyukov#if TSAN_DEBUG && !TSAN_GO 1317ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid DeadlockDetector::Lock(MutexType t) { 132b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov // Printf("LOCK %d @%zu\n", t, seq_ + 1); 133ad9da372f962495b3487685232d09390be841b1cDmitry Vyukov CHECK_GT(t, MutexTypeInvalid); 134ad9da372f962495b3487685232d09390be841b1cDmitry Vyukov CHECK_LT(t, MutexTypeCount); 1357ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 max_seq = 0; 1367ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 max_idx = MutexTypeInvalid; 1377ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (int i = 0; i != MutexTypeCount; i++) { 1387ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (locked_[i] == 0) 1397ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany continue; 1407ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK_NE(locked_[i], max_seq); 1417ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (max_seq < locked_[i]) { 1427ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany max_seq = locked_[i]; 1437ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany max_idx = i; 1447ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1457ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1467ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany locked_[t] = ++seq_; 1477ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (max_idx == MutexTypeInvalid) 1487ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 149b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov // Printf(" last %d @%zu\n", max_idx, max_seq); 1507ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (!CanLockAdj[max_idx][t]) { 151b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("ThreadSanitizer: internal deadlock detected\n"); 152b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov Printf("ThreadSanitizer: can't lock %d while under %zu\n", 153e954101f6602ac181a2c3accfbbad0ae51b0bf7cAlexey Samsonov t, (uptr)max_idx); 154ff35f1d82b4f145b3477ef27a7a2e7b63c486988Dmitry Vyukov CHECK(0); 1557ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1567ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 1577ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1587ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid DeadlockDetector::Unlock(MutexType t) { 159b1fe3021eca0843e37878d224ee7f32e32f40d99Alexey Samsonov // Printf("UNLO %d @%zu #%zu\n", t, seq_, locked_[t]); 1607ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK(locked_[t]); 1617ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany locked_[t] = 0; 1627ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 163d62237995d0fc50697e375ea50f015e996162884Dmitry Vyukov#endif 1647ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1657ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyconst uptr kUnlocked = 0; 1667ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyconst uptr kWriteLock = 1; 1677ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyconst uptr kReadLock = 2; 1687ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1697ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyclass Backoff { 1707ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany public: 1717ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany Backoff() 1727ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany : iter_() { 1737ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1747ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1757ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany bool Do() { 1767ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (iter_++ < kActiveSpinIters) 1777ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany proc_yield(kActiveSpinCnt); 1787ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany else 1790969bcf2c936126b1f6e636b978aade8fc207437Alexey Samsonov internal_sched_yield(); 1807ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return true; 1817ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1827ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1837ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 Contention() const { 1847ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 active = iter_ % kActiveSpinIters; 1857ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 passive = iter_ - active; 1867ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return active + 10 * passive; 1877ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 1887ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1897ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany private: 1907ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany int iter_; 1917ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany static const int kActiveSpinIters = 10; 1927ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany static const int kActiveSpinCnt = 20; 1937ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany}; 1947ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 1957ac41484ea322e0ea5774df681660269f5dc321eKostya SerebryanyMutex::Mutex(MutexType type, StatType stat_type) { 1967ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK_GT(type, MutexTypeInvalid); 1977ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK_LT(type, MutexTypeCount); 1987ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#if TSAN_DEBUG 1997ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany type_ = type; 2007ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 2017ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#if TSAN_COLLECT_STATS 2027ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany stat_type_ = stat_type; 2037ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 2047ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany atomic_store(&state_, kUnlocked, memory_order_relaxed); 2057ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2067ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2077ac41484ea322e0ea5774df681660269f5dc321eKostya SerebryanyMutex::~Mutex() { 2087ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany CHECK_EQ(atomic_load(&state_, memory_order_relaxed), kUnlocked); 2097ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2107ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid Mutex::Lock() { 212b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov#if TSAN_DEBUG && !TSAN_GO 2137ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany cur_thread()->deadlock_detector.Lock(type_); 2147ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 2157ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr cmp = kUnlocked; 2167ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (atomic_compare_exchange_strong(&state_, &cmp, kWriteLock, 2177ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany memory_order_acquire)) 2187ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 2197ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (Backoff backoff; backoff.Do();) { 2207ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (atomic_load(&state_, memory_order_relaxed) == kUnlocked) { 2217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany cmp = kUnlocked; 2227ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (atomic_compare_exchange_weak(&state_, &cmp, kWriteLock, 2237ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany memory_order_acquire)) { 2247ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#if TSAN_COLLECT_STATS 2257ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(cur_thread(), stat_type_, backoff.Contention()); 2267ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 2277ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 2287ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 2297ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 2307ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 2317ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2327ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2337ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid Mutex::Unlock() { 2347ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr prev = atomic_fetch_sub(&state_, kWriteLock, memory_order_release); 2357ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany (void)prev; 2367ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK_NE(prev & kWriteLock, 0); 237b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov#if TSAN_DEBUG && !TSAN_GO 2387ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany cur_thread()->deadlock_detector.Unlock(type_); 2397ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 2407ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2417ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2427ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid Mutex::ReadLock() { 243b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov#if TSAN_DEBUG && !TSAN_GO 2447ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany cur_thread()->deadlock_detector.Lock(type_); 2457ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 2467ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr prev = atomic_fetch_add(&state_, kReadLock, memory_order_acquire); 2477ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if ((prev & kWriteLock) == 0) 2487ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 2497ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany for (Backoff backoff; backoff.Do();) { 2507ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany prev = atomic_load(&state_, memory_order_acquire); 2517ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if ((prev & kWriteLock) == 0) { 2527ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#if TSAN_COLLECT_STATS 2537ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(cur_thread(), stat_type_, backoff.Contention()); 2547ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 2557ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 2567ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 2577ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 2587ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2597ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2607ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid Mutex::ReadUnlock() { 2617ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany uptr prev = atomic_fetch_sub(&state_, kReadLock, memory_order_release); 2627ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany (void)prev; 2637ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK_EQ(prev & kWriteLock, 0); 2647ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany DCHECK_GT(prev & ~kWriteLock, 0); 265b78caa645f70eff2f037f1f8bb43ca9129dbd8d9Dmitry Vyukov#if TSAN_DEBUG && !TSAN_GO 2667ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany cur_thread()->deadlock_detector.Unlock(type_); 2677ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#endif 2687ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2697ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 270ff35f1d82b4f145b3477ef27a7a2e7b63c486988Dmitry Vyukovvoid Mutex::CheckLocked() { 271ff35f1d82b4f145b3477ef27a7a2e7b63c486988Dmitry Vyukov CHECK_NE(atomic_load(&state_, memory_order_relaxed), 0); 272ff35f1d82b4f145b3477ef27a7a2e7b63c486988Dmitry Vyukov} 273ff35f1d82b4f145b3477ef27a7a2e7b63c486988Dmitry Vyukov 2747ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} // namespace __tsan 275