1c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov//===-- sanitizer_mutex.h ---------------------------------------*- C++ -*-===// 2c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov// 3c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov// The LLVM Compiler Infrastructure 4c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov// 5c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov// This file is distributed under the University of Illinois Open Source 6c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov// License. See LICENSE.TXT for details. 7c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov// 8c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov//===----------------------------------------------------------------------===// 9a3eca8192505f4796194dd24eb051019f402de99Dmitry Vyukov// 10a3eca8192505f4796194dd24eb051019f402de99Dmitry Vyukov// This file is a part of ThreadSanitizer/AddressSanitizer runtime. 11a3eca8192505f4796194dd24eb051019f402de99Dmitry Vyukov// 12a3eca8192505f4796194dd24eb051019f402de99Dmitry Vyukov//===----------------------------------------------------------------------===// 13c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov 14c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov#ifndef SANITIZER_MUTEX_H 15c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov#define SANITIZER_MUTEX_H 16c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov 17c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov#include "sanitizer_atomic.h" 188d993186d84cd62b93e1d16ee2afcc6d14629dd1Dmitry Vyukov#include "sanitizer_internal_defs.h" 198d993186d84cd62b93e1d16ee2afcc6d14629dd1Dmitry Vyukov#include "sanitizer_libc.h" 20c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov 21c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukovnamespace __sanitizer { 22c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov 231b37017f0216d0b8f3ae3a7dea8b3cc20d74db25Dmitry Vyukovclass StaticSpinMutex { 24a3eca8192505f4796194dd24eb051019f402de99Dmitry Vyukov public: 251b37017f0216d0b8f3ae3a7dea8b3cc20d74db25Dmitry Vyukov void Init() { 26a3eca8192505f4796194dd24eb051019f402de99Dmitry Vyukov atomic_store(&state_, 0, memory_order_relaxed); 27a3eca8192505f4796194dd24eb051019f402de99Dmitry Vyukov } 28a3eca8192505f4796194dd24eb051019f402de99Dmitry Vyukov 29a3eca8192505f4796194dd24eb051019f402de99Dmitry Vyukov void Lock() { 30a61ec8172611a75d80e86355352c7b2dd3cb8125Dmitry Vyukov if (TryLock()) 318d993186d84cd62b93e1d16ee2afcc6d14629dd1Dmitry Vyukov return; 328d993186d84cd62b93e1d16ee2afcc6d14629dd1Dmitry Vyukov LockSlow(); 33a3eca8192505f4796194dd24eb051019f402de99Dmitry Vyukov } 34a3eca8192505f4796194dd24eb051019f402de99Dmitry Vyukov 35a61ec8172611a75d80e86355352c7b2dd3cb8125Dmitry Vyukov bool TryLock() { 36a61ec8172611a75d80e86355352c7b2dd3cb8125Dmitry Vyukov return atomic_exchange(&state_, 1, memory_order_acquire) == 0; 37a61ec8172611a75d80e86355352c7b2dd3cb8125Dmitry Vyukov } 38a61ec8172611a75d80e86355352c7b2dd3cb8125Dmitry Vyukov 39a3eca8192505f4796194dd24eb051019f402de99Dmitry Vyukov void Unlock() { 40a3eca8192505f4796194dd24eb051019f402de99Dmitry Vyukov atomic_store(&state_, 0, memory_order_release); 41a3eca8192505f4796194dd24eb051019f402de99Dmitry Vyukov } 42a3eca8192505f4796194dd24eb051019f402de99Dmitry Vyukov 438f0c5bdd9650256501bad9fc5dedc977f4ca2247Peter Collingbourne void CheckLocked() { 448fa4edcb169ab77355c77e065fc5387085787a8aKostya Serebryany CHECK_EQ(atomic_load(&state_, memory_order_relaxed), 1); 458fa4edcb169ab77355c77e065fc5387085787a8aKostya Serebryany } 468fa4edcb169ab77355c77e065fc5387085787a8aKostya Serebryany 47a3eca8192505f4796194dd24eb051019f402de99Dmitry Vyukov private: 48a3eca8192505f4796194dd24eb051019f402de99Dmitry Vyukov atomic_uint8_t state_; 49a3eca8192505f4796194dd24eb051019f402de99Dmitry Vyukov 508d993186d84cd62b93e1d16ee2afcc6d14629dd1Dmitry Vyukov void NOINLINE LockSlow() { 518d993186d84cd62b93e1d16ee2afcc6d14629dd1Dmitry Vyukov for (int i = 0;; i++) { 528d993186d84cd62b93e1d16ee2afcc6d14629dd1Dmitry Vyukov if (i < 10) 538d993186d84cd62b93e1d16ee2afcc6d14629dd1Dmitry Vyukov proc_yield(10); 548d993186d84cd62b93e1d16ee2afcc6d14629dd1Dmitry Vyukov else 558d993186d84cd62b93e1d16ee2afcc6d14629dd1Dmitry Vyukov internal_sched_yield(); 568d993186d84cd62b93e1d16ee2afcc6d14629dd1Dmitry Vyukov if (atomic_load(&state_, memory_order_relaxed) == 0 578d993186d84cd62b93e1d16ee2afcc6d14629dd1Dmitry Vyukov && atomic_exchange(&state_, 1, memory_order_acquire) == 0) 588d993186d84cd62b93e1d16ee2afcc6d14629dd1Dmitry Vyukov return; 598d993186d84cd62b93e1d16ee2afcc6d14629dd1Dmitry Vyukov } 608d993186d84cd62b93e1d16ee2afcc6d14629dd1Dmitry Vyukov } 611b37017f0216d0b8f3ae3a7dea8b3cc20d74db25Dmitry Vyukov}; 628d993186d84cd62b93e1d16ee2afcc6d14629dd1Dmitry Vyukov 631b37017f0216d0b8f3ae3a7dea8b3cc20d74db25Dmitry Vyukovclass SpinMutex : public StaticSpinMutex { 641b37017f0216d0b8f3ae3a7dea8b3cc20d74db25Dmitry Vyukov public: 651b37017f0216d0b8f3ae3a7dea8b3cc20d74db25Dmitry Vyukov SpinMutex() { 661b37017f0216d0b8f3ae3a7dea8b3cc20d74db25Dmitry Vyukov Init(); 671b37017f0216d0b8f3ae3a7dea8b3cc20d74db25Dmitry Vyukov } 681b37017f0216d0b8f3ae3a7dea8b3cc20d74db25Dmitry Vyukov 691b37017f0216d0b8f3ae3a7dea8b3cc20d74db25Dmitry Vyukov private: 70a3eca8192505f4796194dd24eb051019f402de99Dmitry Vyukov SpinMutex(const SpinMutex&); 71a3eca8192505f4796194dd24eb051019f402de99Dmitry Vyukov void operator=(const SpinMutex&); 72a3eca8192505f4796194dd24eb051019f402de99Dmitry Vyukov}; 73a3eca8192505f4796194dd24eb051019f402de99Dmitry Vyukov 74f4f51f2cc6fa936f0c65577f82e6b62989d546eeDmitry Vyukovclass BlockingMutex { 75f4f51f2cc6fa936f0c65577f82e6b62989d546eeDmitry Vyukov public: 76f4f51f2cc6fa936f0c65577f82e6b62989d546eeDmitry Vyukov explicit BlockingMutex(LinkerInitialized); 7793af5948d3e0c5bdc396f432dd0ae782f499c449Alexey Samsonov BlockingMutex(); 78f4f51f2cc6fa936f0c65577f82e6b62989d546eeDmitry Vyukov void Lock(); 79f4f51f2cc6fa936f0c65577f82e6b62989d546eeDmitry Vyukov void Unlock(); 80ce700979f644c790c2d9d80f5cc2a1ada0380284Alexey Samsonov void CheckLocked(); 81f4f51f2cc6fa936f0c65577f82e6b62989d546eeDmitry Vyukov private: 82f4f51f2cc6fa936f0c65577f82e6b62989d546eeDmitry Vyukov uptr opaque_storage_[10]; 8348526014de78e21f7ed027a60670016bc7f5d292Dmitry Vyukov uptr owner_; // for debugging 84f4f51f2cc6fa936f0c65577f82e6b62989d546eeDmitry Vyukov}; 85f4f51f2cc6fa936f0c65577f82e6b62989d546eeDmitry Vyukov 862d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// Reader-writer spin mutex. 872d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesclass RWMutex { 882d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines public: 892d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines RWMutex() { 902d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines atomic_store(&state_, kUnlocked, memory_order_relaxed); 912d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 922d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 932d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ~RWMutex() { 942d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines CHECK_EQ(atomic_load(&state_, memory_order_relaxed), kUnlocked); 952d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 962d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 972d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines void Lock() { 982d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines u32 cmp = kUnlocked; 992d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (atomic_compare_exchange_strong(&state_, &cmp, kWriteLock, 1002d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines memory_order_acquire)) 1012d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return; 1022d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines LockSlow(); 1032d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 1042d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 1052d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines void Unlock() { 1062d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines u32 prev = atomic_fetch_sub(&state_, kWriteLock, memory_order_release); 1072d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines DCHECK_NE(prev & kWriteLock, 0); 1082d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines (void)prev; 1092d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 1102d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 1112d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines void ReadLock() { 1122d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines u32 prev = atomic_fetch_add(&state_, kReadLock, memory_order_acquire); 1132d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if ((prev & kWriteLock) == 0) 1142d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return; 1152d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines ReadLockSlow(); 1162d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 1172d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 1182d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines void ReadUnlock() { 1192d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines u32 prev = atomic_fetch_sub(&state_, kReadLock, memory_order_release); 1202d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines DCHECK_EQ(prev & kWriteLock, 0); 1212d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines DCHECK_GT(prev & ~kWriteLock, 0); 1222d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines (void)prev; 1232d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 1242d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 1252d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines void CheckLocked() { 1262d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines CHECK_NE(atomic_load(&state_, memory_order_relaxed), kUnlocked); 1272d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 1282d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 1292d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines private: 1302d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines atomic_uint32_t state_; 1312d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 1322d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines enum { 1332d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines kUnlocked = 0, 1342d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines kWriteLock = 1, 1352d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines kReadLock = 2 1362d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines }; 1372d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 1382d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines void NOINLINE LockSlow() { 1392d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines for (int i = 0;; i++) { 1402d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (i < 10) 1412d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines proc_yield(10); 1422d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines else 1432d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines internal_sched_yield(); 1442d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines u32 cmp = atomic_load(&state_, memory_order_relaxed); 1452d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (cmp == kUnlocked && 1462d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines atomic_compare_exchange_weak(&state_, &cmp, kWriteLock, 1472d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines memory_order_acquire)) 1482d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return; 1492d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 1502d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 1512d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 1522d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines void NOINLINE ReadLockSlow() { 1532d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines for (int i = 0;; i++) { 1542d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (i < 10) 1552d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines proc_yield(10); 1562d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines else 1572d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines internal_sched_yield(); 1582d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines u32 prev = atomic_load(&state_, memory_order_acquire); 1592d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if ((prev & kWriteLock) == 0) 1602d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return; 1612d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 1622d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 1632d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 1642d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines RWMutex(const RWMutex&); 1652d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines void operator = (const RWMutex&); 1662d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines}; 1672d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 168c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukovtemplate<typename MutexType> 169c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukovclass GenericScopedLock { 170c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov public: 171c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov explicit GenericScopedLock(MutexType *mu) 172c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov : mu_(mu) { 173c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov mu_->Lock(); 174c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov } 175c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov 176c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov ~GenericScopedLock() { 177c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov mu_->Unlock(); 178c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov } 179c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov 180c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov private: 181c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov MutexType *mu_; 182c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov 183c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov GenericScopedLock(const GenericScopedLock&); 184c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov void operator=(const GenericScopedLock&); 185c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov}; 186c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov 187c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukovtemplate<typename MutexType> 188c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukovclass GenericScopedReadLock { 189c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov public: 190c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov explicit GenericScopedReadLock(MutexType *mu) 191c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov : mu_(mu) { 192c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov mu_->ReadLock(); 193c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov } 194c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov 195c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov ~GenericScopedReadLock() { 196c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov mu_->ReadUnlock(); 197c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov } 198c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov 199c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov private: 200c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov MutexType *mu_; 201c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov 202c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov GenericScopedReadLock(const GenericScopedReadLock&); 203c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov void operator=(const GenericScopedReadLock&); 204c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov}; 205c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov 2061b37017f0216d0b8f3ae3a7dea8b3cc20d74db25Dmitry Vyukovtypedef GenericScopedLock<StaticSpinMutex> SpinMutexLock; 207f4f51f2cc6fa936f0c65577f82e6b62989d546eeDmitry Vyukovtypedef GenericScopedLock<BlockingMutex> BlockingMutexLock; 2082d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinestypedef GenericScopedLock<RWMutex> RWMutexLock; 2092d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinestypedef GenericScopedReadLock<RWMutex> RWMutexReadLock; 210e088e16006e2b3fc5739be675b3243e18f30ad01Dmitry Vyukov 211c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov} // namespace __sanitizer 212c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov 213c04d8b39795b8faab2c172da59462f4e54823533Dmitry Vyukov#endif // SANITIZER_MUTEX_H 214