tsan_interface_atomic.cc revision f2c29a5dc56feed66e81a7fb3f03ef2c5ea76870
14b562cf889bc59e1914dd2c5d9fbd7e7bfa1ad77Argyrios Kyrtzidis//===-- tsan_interface_atomic.cc ------------------------------------------===// 24b562cf889bc59e1914dd2c5d9fbd7e7bfa1ad77Argyrios Kyrtzidis// 34b562cf889bc59e1914dd2c5d9fbd7e7bfa1ad77Argyrios Kyrtzidis// The LLVM Compiler Infrastructure 44b562cf889bc59e1914dd2c5d9fbd7e7bfa1ad77Argyrios Kyrtzidis// 54b562cf889bc59e1914dd2c5d9fbd7e7bfa1ad77Argyrios Kyrtzidis// This file is distributed under the University of Illinois Open Source 64b562cf889bc59e1914dd2c5d9fbd7e7bfa1ad77Argyrios Kyrtzidis// License. See LICENSE.TXT for details. 74b562cf889bc59e1914dd2c5d9fbd7e7bfa1ad77Argyrios Kyrtzidis// 84b562cf889bc59e1914dd2c5d9fbd7e7bfa1ad77Argyrios Kyrtzidis//===----------------------------------------------------------------------===// 94b562cf889bc59e1914dd2c5d9fbd7e7bfa1ad77Argyrios Kyrtzidis// 104b562cf889bc59e1914dd2c5d9fbd7e7bfa1ad77Argyrios Kyrtzidis// This file is a part of ThreadSanitizer (TSan), a race detector. 114b562cf889bc59e1914dd2c5d9fbd7e7bfa1ad77Argyrios Kyrtzidis// 124b562cf889bc59e1914dd2c5d9fbd7e7bfa1ad77Argyrios Kyrtzidis//===----------------------------------------------------------------------===// 134b562cf889bc59e1914dd2c5d9fbd7e7bfa1ad77Argyrios Kyrtzidis 140853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidis// ThreadSanitizer atomic operations are based on C++11/C1x standards. 150853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidis// For background see C++11 standard. A slightly older, publically 160853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidis// available draft of the standard (not entirely up-to-date, but close enough 174ae8f298b1ea51b4c2234f9148e2e4349c9bdd23Douglas Gregor// for casual browsing) is available here: 1831b87d8006d4863dd9b17e515ac720941efc38e3Daniel Dunbar// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf 1928019772db70d4547be05a042eb950bc910f134fDouglas Gregor// The following page contains more background information: 200853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidis// http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/ 2136c4464ba6cfc2a63dc67c493ef2f5ab2aea09ccSteve Naroff 22f96b524306ccfa623235d375deee79637bd38f29Steve Naroff#include "sanitizer_common/sanitizer_placement_new.h" 23a88084b78fd4ca5d3d858c14b02414f8cc399f02Douglas Gregor#include "sanitizer_common/sanitizer_stacktrace.h" 24313e26c4e81f0e467490a530548450f4c824a6c4Douglas Gregor#include "tsan_interface_atomic.h" 25788f5a1242c04762f91eaa7565c07b9865846d88Douglas Gregor#include "tsan_flags.h" 260853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidis#include "tsan_rtl.h" 27f772d1e2a5688572d07f42896a50ac57a4a41fe8Daniel Dunbar 28f772d1e2a5688572d07f42896a50ac57a4a41fe8Daniel Dunbarusing namespace __tsan; // NOLINT 294db64a461cb3442934afe43c83ed3f17f7c11c1dDouglas Gregor 304db64a461cb3442934afe43c83ed3f17f7c11c1dDouglas Gregor#define SCOPED_ATOMIC(func, ...) \ 314db64a461cb3442934afe43c83ed3f17f7c11c1dDouglas Gregor const uptr callpc = (uptr)__builtin_return_address(0); \ 324db64a461cb3442934afe43c83ed3f17f7c11c1dDouglas Gregor uptr pc = __sanitizer::StackTrace::GetCurrentPc(); \ 334db64a461cb3442934afe43c83ed3f17f7c11c1dDouglas Gregor mo = ConvertOrder(mo); \ 340853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidis mo = flags()->force_seq_cst_atomics ? (morder)mo_seq_cst : mo; \ 350853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidis ThreadState *const thr = cur_thread(); \ 36521bf9c529e653ab28896d027352d3e16e2672d5Daniel Dunbar AtomicStatInc(thr, sizeof(*a), mo, StatAtomic##func); \ 37521bf9c529e653ab28896d027352d3e16e2672d5Daniel Dunbar ScopedAtomic sa(thr, callpc, a, mo, __FUNCTION__); \ 38521bf9c529e653ab28896d027352d3e16e2672d5Daniel Dunbar return Atomic##func(thr, pc, __VA_ARGS__); \ 39521bf9c529e653ab28896d027352d3e16e2672d5Daniel Dunbar/**/ 40521bf9c529e653ab28896d027352d3e16e2672d5Daniel Dunbar 41521bf9c529e653ab28896d027352d3e16e2672d5Daniel Dunbar// Some shortcuts. 42521bf9c529e653ab28896d027352d3e16e2672d5Daniel Dunbartypedef __tsan_memory_order morder; 43521bf9c529e653ab28896d027352d3e16e2672d5Daniel Dunbartypedef __tsan_atomic8 a8; 44521bf9c529e653ab28896d027352d3e16e2672d5Daniel Dunbartypedef __tsan_atomic16 a16; 45521bf9c529e653ab28896d027352d3e16e2672d5Daniel Dunbartypedef __tsan_atomic32 a32; 460853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidistypedef __tsan_atomic64 a64; 47f96b524306ccfa623235d375deee79637bd38f29Steve Narofftypedef __tsan_atomic128 a128; 48f96b524306ccfa623235d375deee79637bd38f29Steve Naroffconst morder mo_relaxed = __tsan_memory_order_relaxed; 490853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidisconst morder mo_consume = __tsan_memory_order_consume; 500853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidisconst morder mo_acquire = __tsan_memory_order_acquire; 510853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidisconst morder mo_release = __tsan_memory_order_release; 52788f5a1242c04762f91eaa7565c07b9865846d88Douglas Gregorconst morder mo_acq_rel = __tsan_memory_order_acq_rel; 53788f5a1242c04762f91eaa7565c07b9865846d88Douglas Gregorconst morder mo_seq_cst = __tsan_memory_order_seq_cst; 54788f5a1242c04762f91eaa7565c07b9865846d88Douglas Gregor 55788f5a1242c04762f91eaa7565c07b9865846d88Douglas Gregorclass ScopedAtomic { 5628019772db70d4547be05a042eb950bc910f134fDouglas Gregor public: 57405634b215f19eec7183bd8005e34aa5a02f64a1Douglas Gregor ScopedAtomic(ThreadState *thr, uptr pc, const volatile void *a, 58405634b215f19eec7183bd8005e34aa5a02f64a1Douglas Gregor morder mo, const char *func) 590853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidis : thr_(thr) { 600853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidis CHECK_EQ(thr_->in_rtl, 0); 610853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidis ProcessPendingSignals(thr); 620853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidis FuncEntry(thr_, pc); 634ae8f298b1ea51b4c2234f9148e2e4349c9bdd23Douglas Gregor DPrintf("#%d: %s(%p, %d)\n", thr_->tid, func, a, mo); 64807b06157a1a5c050520fc194d32f16d22d423a8Daniel Dunbar thr_->in_rtl++; 65807b06157a1a5c050520fc194d32f16d22d423a8Daniel Dunbar } 66807b06157a1a5c050520fc194d32f16d22d423a8Daniel Dunbar ~ScopedAtomic() { 67807b06157a1a5c050520fc194d32f16d22d423a8Daniel Dunbar thr_->in_rtl--; 687d1d49d2971b20a97b3c2a301470b9eaaa130137Douglas Gregor CHECK_EQ(thr_->in_rtl, 0); 697d1d49d2971b20a97b3c2a301470b9eaaa130137Douglas Gregor FuncExit(thr_); 707d1d49d2971b20a97b3c2a301470b9eaaa130137Douglas Gregor } 717d1d49d2971b20a97b3c2a301470b9eaaa130137Douglas Gregor private: 72c7822dbf3c01a2a5f837cff82ba7889ea755dacaDaniel Dunbar ThreadState *thr_; 73f772d1e2a5688572d07f42896a50ac57a4a41fe8Daniel Dunbar}; 74c7822dbf3c01a2a5f837cff82ba7889ea755dacaDaniel Dunbar 75c7822dbf3c01a2a5f837cff82ba7889ea755dacaDaniel Dunbarstatic void AtomicStatInc(ThreadState *thr, uptr size, morder mo, StatType t) { 76f772d1e2a5688572d07f42896a50ac57a4a41fe8Daniel Dunbar StatInc(thr, StatAtomic); 77f772d1e2a5688572d07f42896a50ac57a4a41fe8Daniel Dunbar StatInc(thr, t); 78f772d1e2a5688572d07f42896a50ac57a4a41fe8Daniel Dunbar StatInc(thr, size == 1 ? StatAtomic1 79f772d1e2a5688572d07f42896a50ac57a4a41fe8Daniel Dunbar : size == 2 ? StatAtomic2 80f772d1e2a5688572d07f42896a50ac57a4a41fe8Daniel Dunbar : size == 4 ? StatAtomic4 81f772d1e2a5688572d07f42896a50ac57a4a41fe8Daniel Dunbar : size == 8 ? StatAtomic8 82f772d1e2a5688572d07f42896a50ac57a4a41fe8Daniel Dunbar : StatAtomic16); 83f772d1e2a5688572d07f42896a50ac57a4a41fe8Daniel Dunbar StatInc(thr, mo == mo_relaxed ? StatAtomicRelaxed 84f772d1e2a5688572d07f42896a50ac57a4a41fe8Daniel Dunbar : mo == mo_consume ? StatAtomicConsume 8568d40e2d16b9fadba386853d6bbb60089291fdc5Daniel Dunbar : mo == mo_acquire ? StatAtomicAcquire 8668d40e2d16b9fadba386853d6bbb60089291fdc5Daniel Dunbar : mo == mo_release ? StatAtomicRelease 8768d40e2d16b9fadba386853d6bbb60089291fdc5Daniel Dunbar : mo == mo_acq_rel ? StatAtomicAcq_Rel 88f96b524306ccfa623235d375deee79637bd38f29Steve Naroff : StatAtomicSeq_Cst); 89f96b524306ccfa623235d375deee79637bd38f29Steve Naroff} 90f772d1e2a5688572d07f42896a50ac57a4a41fe8Daniel Dunbar 91a88084b78fd4ca5d3d858c14b02414f8cc399f02Douglas Gregorstatic bool IsLoadOrder(morder mo) { 92a88084b78fd4ca5d3d858c14b02414f8cc399f02Douglas Gregor return mo == mo_relaxed || mo == mo_consume 93405634b215f19eec7183bd8005e34aa5a02f64a1Douglas Gregor || mo == mo_acquire || mo == mo_seq_cst; 94a88084b78fd4ca5d3d858c14b02414f8cc399f02Douglas Gregor} 95313e26c4e81f0e467490a530548450f4c824a6c4Douglas Gregor 96313e26c4e81f0e467490a530548450f4c824a6c4Douglas Gregorstatic bool IsStoreOrder(morder mo) { 97313e26c4e81f0e467490a530548450f4c824a6c4Douglas Gregor return mo == mo_relaxed || mo == mo_release || mo == mo_seq_cst; 98bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregor} 99788f5a1242c04762f91eaa7565c07b9865846d88Douglas Gregor 100788f5a1242c04762f91eaa7565c07b9865846d88Douglas Gregorstatic bool IsReleaseOrder(morder mo) { 101788f5a1242c04762f91eaa7565c07b9865846d88Douglas Gregor return mo == mo_release || mo == mo_acq_rel || mo == mo_seq_cst; 102788f5a1242c04762f91eaa7565c07b9865846d88Douglas Gregor} 103788f5a1242c04762f91eaa7565c07b9865846d88Douglas Gregor 104788f5a1242c04762f91eaa7565c07b9865846d88Douglas Gregorstatic bool IsAcquireOrder(morder mo) { 105788f5a1242c04762f91eaa7565c07b9865846d88Douglas Gregor return mo == mo_consume || mo == mo_acquire 106788f5a1242c04762f91eaa7565c07b9865846d88Douglas Gregor || mo == mo_acq_rel || mo == mo_seq_cst; 107788f5a1242c04762f91eaa7565c07b9865846d88Douglas Gregor} 108bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregor 109bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregorstatic bool IsAcqRelOrder(morder mo) { 110bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregor return mo == mo_acq_rel || mo == mo_seq_cst; 111bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregor} 112bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregor 113bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregorstatic morder ConvertOrder(morder mo) { 114bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregor if (mo > (morder)100500) { 115bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregor mo = morder(mo - 100500); 116bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregor if (mo == morder(1 << 0)) 117313e26c4e81f0e467490a530548450f4c824a6c4Douglas Gregor mo = mo_relaxed; 11831b87d8006d4863dd9b17e515ac720941efc38e3Daniel Dunbar else if (mo == morder(1 << 1)) 11931b87d8006d4863dd9b17e515ac720941efc38e3Daniel Dunbar mo = mo_consume; 120bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregor else if (mo == morder(1 << 2)) 1213687e9d3a5dbfa9963af02a49a2b139d91310813Douglas Gregor mo = mo_acquire; 1228b96253907c47141af0b7b2a44a368748d006a87Douglas Gregor else if (mo == morder(1 << 3)) 1230853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidis mo = mo_release; 124bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregor else if (mo == morder(1 << 4)) 125bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregor mo = mo_acq_rel; 126bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregor else if (mo == morder(1 << 5)) 127bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregor mo = mo_seq_cst; 128bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregor } 129bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregor CHECK_GE(mo, mo_relaxed); 130bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregor CHECK_LE(mo, mo_seq_cst); 131bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregor return mo; 132bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregor} 133bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregor 134bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregortemplate<typename T> T func_xchg(volatile T *v, T op) { 135bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregor T res = __sync_lock_test_and_set(v, op); 136bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregor // __sync_lock_test_and_set does not contain full barrier. 137bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregor __sync_synchronize(); 138bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregor return res; 139bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregor} 140bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregor 141bdf6062bc10aa3b73b16402b440b8073310acd06Douglas Gregortemplate<typename T> T func_add(volatile T *v, T op) { 1420853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidis return __sync_fetch_and_add(v, op); 1430853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidis} 144c7822dbf3c01a2a5f837cff82ba7889ea755dacaDaniel Dunbar 145c7822dbf3c01a2a5f837cff82ba7889ea755dacaDaniel Dunbartemplate<typename T> T func_sub(volatile T *v, T op) { 1463687e9d3a5dbfa9963af02a49a2b139d91310813Douglas Gregor return __sync_fetch_and_sub(v, op); 1473687e9d3a5dbfa9963af02a49a2b139d91310813Douglas Gregor} 1483687e9d3a5dbfa9963af02a49a2b139d91310813Douglas Gregor 149405634b215f19eec7183bd8005e34aa5a02f64a1Douglas Gregortemplate<typename T> T func_and(volatile T *v, T op) { 150405634b215f19eec7183bd8005e34aa5a02f64a1Douglas Gregor return __sync_fetch_and_and(v, op); 1510853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidis} 1520853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidis 1530853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidistemplate<typename T> T func_or(volatile T *v, T op) { 1541eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump return __sync_fetch_and_or(v, op); 1550853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidis} 1560853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidis 1570853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidistemplate<typename T> T func_xor(volatile T *v, T op) { 158405634b215f19eec7183bd8005e34aa5a02f64a1Douglas Gregor return __sync_fetch_and_xor(v, op); 159405634b215f19eec7183bd8005e34aa5a02f64a1Douglas Gregor} 160f772d1e2a5688572d07f42896a50ac57a4a41fe8Daniel Dunbar 16177accc11f04ed4ff9afd4e27d430144d4714be56Steve Narofftemplate<typename T> T func_nand(volatile T *v, T op) { 162e19944c93961b7618f4f3f3185f698f46369ea54Steve Naroff // clang does not support __sync_fetch_and_nand. 163b85bca2676b433ae555db09de4dd2823ff13b856Zhongxing Xu T cmp = *v; 164313e26c4e81f0e467490a530548450f4c824a6c4Douglas Gregor for (;;) { 165313e26c4e81f0e467490a530548450f4c824a6c4Douglas Gregor T newv = ~(cmp & op); 166313e26c4e81f0e467490a530548450f4c824a6c4Douglas Gregor T cur = __sync_val_compare_and_swap(v, cmp, newv); 167313e26c4e81f0e467490a530548450f4c824a6c4Douglas Gregor if (cmp == cur) 168313e26c4e81f0e467490a530548450f4c824a6c4Douglas Gregor return cmp; 169313e26c4e81f0e467490a530548450f4c824a6c4Douglas Gregor cmp = cur; 170313e26c4e81f0e467490a530548450f4c824a6c4Douglas Gregor } 1717d1d49d2971b20a97b3c2a301470b9eaaa130137Douglas Gregor} 172f772d1e2a5688572d07f42896a50ac57a4a41fe8Daniel Dunbar 173f96b524306ccfa623235d375deee79637bd38f29Steve Narofftemplate<typename T> T func_cas(volatile T *v, T cmp, T xch) { 174f96b524306ccfa623235d375deee79637bd38f29Steve Naroff return __sync_val_compare_and_swap(v, cmp, xch); 175f772d1e2a5688572d07f42896a50ac57a4a41fe8Daniel Dunbar} 176f772d1e2a5688572d07f42896a50ac57a4a41fe8Daniel Dunbar 177f772d1e2a5688572d07f42896a50ac57a4a41fe8Daniel Dunbar// clang does not support 128-bit atomic ops. 178f772d1e2a5688572d07f42896a50ac57a4a41fe8Daniel Dunbar// Atomic ops are executed under tsan internal mutex, 179f772d1e2a5688572d07f42896a50ac57a4a41fe8Daniel Dunbar// here we assume that the atomic variables are not accessed 180f772d1e2a5688572d07f42896a50ac57a4a41fe8Daniel Dunbar// from non-instrumented code. 181f772d1e2a5688572d07f42896a50ac57a4a41fe8Daniel Dunbar#ifndef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 182f772d1e2a5688572d07f42896a50ac57a4a41fe8Daniel Dunbara128 func_xchg(volatile a128 *v, a128 op) { 183f772d1e2a5688572d07f42896a50ac57a4a41fe8Daniel Dunbar a128 cmp = *v; 184f772d1e2a5688572d07f42896a50ac57a4a41fe8Daniel Dunbar *v = op; 185788f5a1242c04762f91eaa7565c07b9865846d88Douglas Gregor return cmp; 186788f5a1242c04762f91eaa7565c07b9865846d88Douglas Gregor} 187788f5a1242c04762f91eaa7565c07b9865846d88Douglas Gregor 188788f5a1242c04762f91eaa7565c07b9865846d88Douglas Gregora128 func_add(volatile a128 *v, a128 op) { 189788f5a1242c04762f91eaa7565c07b9865846d88Douglas Gregor a128 cmp = *v; 190788f5a1242c04762f91eaa7565c07b9865846d88Douglas Gregor *v = cmp + op; 191a88084b78fd4ca5d3d858c14b02414f8cc399f02Douglas Gregor return cmp; 192405634b215f19eec7183bd8005e34aa5a02f64a1Douglas Gregor} 193405634b215f19eec7183bd8005e34aa5a02f64a1Douglas Gregor 194405634b215f19eec7183bd8005e34aa5a02f64a1Douglas Gregora128 func_sub(volatile a128 *v, a128 op) { 195405634b215f19eec7183bd8005e34aa5a02f64a1Douglas Gregor a128 cmp = *v; 196405634b215f19eec7183bd8005e34aa5a02f64a1Douglas Gregor *v = cmp - op; 197405634b215f19eec7183bd8005e34aa5a02f64a1Douglas Gregor return cmp; 198405634b215f19eec7183bd8005e34aa5a02f64a1Douglas Gregor} 199405634b215f19eec7183bd8005e34aa5a02f64a1Douglas Gregor 200405634b215f19eec7183bd8005e34aa5a02f64a1Douglas Gregora128 func_and(volatile a128 *v, a128 op) { 201405634b215f19eec7183bd8005e34aa5a02f64a1Douglas Gregor a128 cmp = *v; 202405634b215f19eec7183bd8005e34aa5a02f64a1Douglas Gregor *v = cmp & op; 203a88084b78fd4ca5d3d858c14b02414f8cc399f02Douglas Gregor return cmp; 204a88084b78fd4ca5d3d858c14b02414f8cc399f02Douglas Gregor} 2054db64a461cb3442934afe43c83ed3f17f7c11c1dDouglas Gregor 2064db64a461cb3442934afe43c83ed3f17f7c11c1dDouglas Gregora128 func_or(volatile a128 *v, a128 op) { 2074db64a461cb3442934afe43c83ed3f17f7c11c1dDouglas Gregor a128 cmp = *v; 2084db64a461cb3442934afe43c83ed3f17f7c11c1dDouglas Gregor *v = cmp | op; 2090853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidis return cmp; 2100853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidis} 21131b87d8006d4863dd9b17e515ac720941efc38e3Daniel Dunbar 2120853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidisa128 func_xor(volatile a128 *v, a128 op) { 2135262fda30b876c8aae95f2eb92e349418d6b14bbDaniel Dunbar a128 cmp = *v; 2145262fda30b876c8aae95f2eb92e349418d6b14bbDaniel Dunbar *v = cmp ^ op; 21531b87d8006d4863dd9b17e515ac720941efc38e3Daniel Dunbar return cmp; 21631b87d8006d4863dd9b17e515ac720941efc38e3Daniel Dunbar} 2170853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidis 21828019772db70d4547be05a042eb950bc910f134fDouglas Gregora128 func_nand(volatile a128 *v, a128 op) { 2195cf48766d626ff6b223acc9d4b7e415ca8480836Ted Kremenek a128 cmp = *v; 2204db64a461cb3442934afe43c83ed3f17f7c11c1dDouglas Gregor *v = ~(cmp & op); 221a88084b78fd4ca5d3d858c14b02414f8cc399f02Douglas Gregor return cmp; 222a88084b78fd4ca5d3d858c14b02414f8cc399f02Douglas Gregor} 223521bf9c529e653ab28896d027352d3e16e2672d5Daniel Dunbar 224521bf9c529e653ab28896d027352d3e16e2672d5Daniel Dunbara128 func_cas(volatile a128 *v, a128 cmp, a128 xch) { 225521bf9c529e653ab28896d027352d3e16e2672d5Daniel Dunbar a128 cur = *v; 226521bf9c529e653ab28896d027352d3e16e2672d5Daniel Dunbar if (cur == cmp) 227521bf9c529e653ab28896d027352d3e16e2672d5Daniel Dunbar *v = xch; 228f7acc37450d59ef751df73acb91de73850cc6517Daniel Dunbar return cur; 229521bf9c529e653ab28896d027352d3e16e2672d5Daniel Dunbar} 2305262fda30b876c8aae95f2eb92e349418d6b14bbDaniel Dunbar#endif 2315262fda30b876c8aae95f2eb92e349418d6b14bbDaniel Dunbar 232521bf9c529e653ab28896d027352d3e16e2672d5Daniel Dunbartemplate<typename T> 233521bf9c529e653ab28896d027352d3e16e2672d5Daniel Dunbarstatic int SizeLog() { 234521bf9c529e653ab28896d027352d3e16e2672d5Daniel Dunbar if (sizeof(T) <= 1) 235f7acc37450d59ef751df73acb91de73850cc6517Daniel Dunbar return kSizeLog1; 23628019772db70d4547be05a042eb950bc910f134fDouglas Gregor else if (sizeof(T) <= 2) 237a88084b78fd4ca5d3d858c14b02414f8cc399f02Douglas Gregor return kSizeLog2; 23894dc8f640ebea52241412512ed48601626edbc58Douglas Gregor else if (sizeof(T) <= 4) 239521bf9c529e653ab28896d027352d3e16e2672d5Daniel Dunbar return kSizeLog4; 2407b55668db7618334cc40011d3c1e128524d89462Daniel Dunbar else 2417b55668db7618334cc40011d3c1e128524d89462Daniel Dunbar return kSizeLog8; 2427b55668db7618334cc40011d3c1e128524d89462Daniel Dunbar // For 16-byte atomics we also use 8-byte memory access, 2437b55668db7618334cc40011d3c1e128524d89462Daniel Dunbar // this leads to false negatives only in very obscure cases. 2447b55668db7618334cc40011d3c1e128524d89462Daniel Dunbar} 2457b55668db7618334cc40011d3c1e128524d89462Daniel Dunbar 2467b55668db7618334cc40011d3c1e128524d89462Daniel Dunbartemplate<typename T> 2475262fda30b876c8aae95f2eb92e349418d6b14bbDaniel Dunbarstatic T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, 2485262fda30b876c8aae95f2eb92e349418d6b14bbDaniel Dunbar morder mo) { 2497b55668db7618334cc40011d3c1e128524d89462Daniel Dunbar CHECK(IsLoadOrder(mo)); 250869824e87940f97b87064db2df2861e82e08a8c6Daniel Dunbar // This fast-path is critical for performance. 2517b55668db7618334cc40011d3c1e128524d89462Daniel Dunbar // Assume the access is atomic. 2527b55668db7618334cc40011d3c1e128524d89462Daniel Dunbar if (!IsAcquireOrder(mo) && sizeof(T) <= sizeof(a)) { 2537b55668db7618334cc40011d3c1e128524d89462Daniel Dunbar MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>()); 2547b55668db7618334cc40011d3c1e128524d89462Daniel Dunbar return *a; 2557b55668db7618334cc40011d3c1e128524d89462Daniel Dunbar } 25628019772db70d4547be05a042eb950bc910f134fDouglas Gregor SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, false); 257869824e87940f97b87064db2df2861e82e08a8c6Daniel Dunbar thr->clock.set(thr->tid, thr->fast_state.epoch()); 2587b55668db7618334cc40011d3c1e128524d89462Daniel Dunbar thr->clock.acquire(&s->clock); 2594db64a461cb3442934afe43c83ed3f17f7c11c1dDouglas Gregor T v = *a; 260a88084b78fd4ca5d3d858c14b02414f8cc399f02Douglas Gregor s->mtx.ReadUnlock(); 26194dc8f640ebea52241412512ed48601626edbc58Douglas Gregor __sync_synchronize(); 2620853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidis MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>()); 2630853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidis return v; 2640853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidis} 2650853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidis 2660853a02c3b04d96a3c432b883e403175c954cd81Argyrios Kyrtzidistemplate<typename T> 267static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v, 268 morder mo) { 269 CHECK(IsStoreOrder(mo)); 270 MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>()); 271 // This fast-path is critical for performance. 272 // Assume the access is atomic. 273 // Strictly saying even relaxed store cuts off release sequence, 274 // so must reset the clock. 275 if (!IsReleaseOrder(mo) && sizeof(T) <= sizeof(a)) { 276 *a = v; 277 return; 278 } 279 __sync_synchronize(); 280 SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, true); 281 thr->clock.set(thr->tid, thr->fast_state.epoch()); 282 thr->clock.ReleaseStore(&s->clock); 283 *a = v; 284 s->mtx.Unlock(); 285 // Trainling memory barrier to provide sequential consistency 286 // for Dekker-like store-load synchronization. 287 __sync_synchronize(); 288} 289 290template<typename T, T (*F)(volatile T *v, T op)> 291static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { 292 MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>()); 293 SyncVar *s = 0; 294 if (mo != mo_relaxed) { 295 s = CTX()->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, true); 296 thr->clock.set(thr->tid, thr->fast_state.epoch()); 297 if (IsAcqRelOrder(mo)) 298 thr->clock.acq_rel(&s->clock); 299 else if (IsReleaseOrder(mo)) 300 thr->clock.release(&s->clock); 301 else if (IsAcquireOrder(mo)) 302 thr->clock.acquire(&s->clock); 303 } 304 v = F(a, v); 305 if (s) 306 s->mtx.Unlock(); 307 return v; 308} 309 310template<typename T> 311static T AtomicExchange(ThreadState *thr, uptr pc, volatile T *a, T v, 312 morder mo) { 313 return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo); 314} 315 316template<typename T> 317static T AtomicFetchAdd(ThreadState *thr, uptr pc, volatile T *a, T v, 318 morder mo) { 319 return AtomicRMW<T, func_add>(thr, pc, a, v, mo); 320} 321 322template<typename T> 323static T AtomicFetchSub(ThreadState *thr, uptr pc, volatile T *a, T v, 324 morder mo) { 325 return AtomicRMW<T, func_sub>(thr, pc, a, v, mo); 326} 327 328template<typename T> 329static T AtomicFetchAnd(ThreadState *thr, uptr pc, volatile T *a, T v, 330 morder mo) { 331 return AtomicRMW<T, func_and>(thr, pc, a, v, mo); 332} 333 334template<typename T> 335static T AtomicFetchOr(ThreadState *thr, uptr pc, volatile T *a, T v, 336 morder mo) { 337 return AtomicRMW<T, func_or>(thr, pc, a, v, mo); 338} 339 340template<typename T> 341static T AtomicFetchXor(ThreadState *thr, uptr pc, volatile T *a, T v, 342 morder mo) { 343 return AtomicRMW<T, func_xor>(thr, pc, a, v, mo); 344} 345 346template<typename T> 347static T AtomicFetchNand(ThreadState *thr, uptr pc, volatile T *a, T v, 348 morder mo) { 349 return AtomicRMW<T, func_nand>(thr, pc, a, v, mo); 350} 351 352template<typename T> 353static bool AtomicCAS(ThreadState *thr, uptr pc, 354 volatile T *a, T *c, T v, morder mo, morder fmo) { 355 (void)fmo; // Unused because llvm does not pass it yet. 356 MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>()); 357 SyncVar *s = 0; 358 if (mo != mo_relaxed) { 359 s = CTX()->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, true); 360 thr->clock.set(thr->tid, thr->fast_state.epoch()); 361 if (IsAcqRelOrder(mo)) 362 thr->clock.acq_rel(&s->clock); 363 else if (IsReleaseOrder(mo)) 364 thr->clock.release(&s->clock); 365 else if (IsAcquireOrder(mo)) 366 thr->clock.acquire(&s->clock); 367 } 368 T cc = *c; 369 T pr = func_cas(a, cc, v); 370 if (s) 371 s->mtx.Unlock(); 372 if (pr == cc) 373 return true; 374 *c = pr; 375 return false; 376} 377 378template<typename T> 379static T AtomicCAS(ThreadState *thr, uptr pc, 380 volatile T *a, T c, T v, morder mo, morder fmo) { 381 AtomicCAS(thr, pc, a, &c, v, mo, fmo); 382 return c; 383} 384 385static void AtomicFence(ThreadState *thr, uptr pc, morder mo) { 386 // FIXME(dvyukov): not implemented. 387 __sync_synchronize(); 388} 389 390a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) { 391 SCOPED_ATOMIC(Load, a, mo); 392} 393 394a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) { 395 SCOPED_ATOMIC(Load, a, mo); 396} 397 398a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) { 399 SCOPED_ATOMIC(Load, a, mo); 400} 401 402a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) { 403 SCOPED_ATOMIC(Load, a, mo); 404} 405 406#if __TSAN_HAS_INT128 407a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) { 408 SCOPED_ATOMIC(Load, a, mo); 409} 410#endif 411 412void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) { 413 SCOPED_ATOMIC(Store, a, v, mo); 414} 415 416void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) { 417 SCOPED_ATOMIC(Store, a, v, mo); 418} 419 420void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) { 421 SCOPED_ATOMIC(Store, a, v, mo); 422} 423 424void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) { 425 SCOPED_ATOMIC(Store, a, v, mo); 426} 427 428#if __TSAN_HAS_INT128 429void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) { 430 SCOPED_ATOMIC(Store, a, v, mo); 431} 432#endif 433 434a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) { 435 SCOPED_ATOMIC(Exchange, a, v, mo); 436} 437 438a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) { 439 SCOPED_ATOMIC(Exchange, a, v, mo); 440} 441 442a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) { 443 SCOPED_ATOMIC(Exchange, a, v, mo); 444} 445 446a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) { 447 SCOPED_ATOMIC(Exchange, a, v, mo); 448} 449 450#if __TSAN_HAS_INT128 451a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) { 452 SCOPED_ATOMIC(Exchange, a, v, mo); 453} 454#endif 455 456a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) { 457 SCOPED_ATOMIC(FetchAdd, a, v, mo); 458} 459 460a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) { 461 SCOPED_ATOMIC(FetchAdd, a, v, mo); 462} 463 464a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) { 465 SCOPED_ATOMIC(FetchAdd, a, v, mo); 466} 467 468a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) { 469 SCOPED_ATOMIC(FetchAdd, a, v, mo); 470} 471 472#if __TSAN_HAS_INT128 473a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) { 474 SCOPED_ATOMIC(FetchAdd, a, v, mo); 475} 476#endif 477 478a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) { 479 SCOPED_ATOMIC(FetchSub, a, v, mo); 480} 481 482a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) { 483 SCOPED_ATOMIC(FetchSub, a, v, mo); 484} 485 486a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) { 487 SCOPED_ATOMIC(FetchSub, a, v, mo); 488} 489 490a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) { 491 SCOPED_ATOMIC(FetchSub, a, v, mo); 492} 493 494#if __TSAN_HAS_INT128 495a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) { 496 SCOPED_ATOMIC(FetchSub, a, v, mo); 497} 498#endif 499 500a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) { 501 SCOPED_ATOMIC(FetchAnd, a, v, mo); 502} 503 504a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) { 505 SCOPED_ATOMIC(FetchAnd, a, v, mo); 506} 507 508a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) { 509 SCOPED_ATOMIC(FetchAnd, a, v, mo); 510} 511 512a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) { 513 SCOPED_ATOMIC(FetchAnd, a, v, mo); 514} 515 516#if __TSAN_HAS_INT128 517a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) { 518 SCOPED_ATOMIC(FetchAnd, a, v, mo); 519} 520#endif 521 522a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) { 523 SCOPED_ATOMIC(FetchOr, a, v, mo); 524} 525 526a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) { 527 SCOPED_ATOMIC(FetchOr, a, v, mo); 528} 529 530a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) { 531 SCOPED_ATOMIC(FetchOr, a, v, mo); 532} 533 534a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) { 535 SCOPED_ATOMIC(FetchOr, a, v, mo); 536} 537 538#if __TSAN_HAS_INT128 539a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) { 540 SCOPED_ATOMIC(FetchOr, a, v, mo); 541} 542#endif 543 544a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) { 545 SCOPED_ATOMIC(FetchXor, a, v, mo); 546} 547 548a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) { 549 SCOPED_ATOMIC(FetchXor, a, v, mo); 550} 551 552a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) { 553 SCOPED_ATOMIC(FetchXor, a, v, mo); 554} 555 556a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) { 557 SCOPED_ATOMIC(FetchXor, a, v, mo); 558} 559 560#if __TSAN_HAS_INT128 561a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) { 562 SCOPED_ATOMIC(FetchXor, a, v, mo); 563} 564#endif 565 566a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) { 567 SCOPED_ATOMIC(FetchNand, a, v, mo); 568} 569 570a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) { 571 SCOPED_ATOMIC(FetchNand, a, v, mo); 572} 573 574a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) { 575 SCOPED_ATOMIC(FetchNand, a, v, mo); 576} 577 578a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) { 579 SCOPED_ATOMIC(FetchNand, a, v, mo); 580} 581 582#if __TSAN_HAS_INT128 583a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) { 584 SCOPED_ATOMIC(FetchNand, a, v, mo); 585} 586#endif 587 588int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v, 589 morder mo, morder fmo) { 590 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 591} 592 593int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v, 594 morder mo, morder fmo) { 595 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 596} 597 598int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v, 599 morder mo, morder fmo) { 600 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 601} 602 603int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v, 604 morder mo, morder fmo) { 605 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 606} 607 608#if __TSAN_HAS_INT128 609int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v, 610 morder mo, morder fmo) { 611 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 612} 613#endif 614 615int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v, 616 morder mo, morder fmo) { 617 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 618} 619 620int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v, 621 morder mo, morder fmo) { 622 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 623} 624 625int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v, 626 morder mo, morder fmo) { 627 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 628} 629 630int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v, 631 morder mo, morder fmo) { 632 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 633} 634 635#if __TSAN_HAS_INT128 636int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v, 637 morder mo, morder fmo) { 638 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 639} 640#endif 641 642a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v, 643 morder mo, morder fmo) { 644 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 645} 646a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v, 647 morder mo, morder fmo) { 648 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 649} 650 651a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v, 652 morder mo, morder fmo) { 653 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 654} 655 656a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v, 657 morder mo, morder fmo) { 658 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 659} 660 661#if __TSAN_HAS_INT128 662a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v, 663 morder mo, morder fmo) { 664 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 665} 666#endif 667 668void __tsan_atomic_thread_fence(morder mo) { 669 char* a = 0; 670 SCOPED_ATOMIC(Fence, mo); 671} 672 673void __tsan_atomic_signal_fence(morder mo) { 674} 675