1603c4be006d8c53905d736bf1f19a49f5ce98276Alexey Samsonov//===-- tsan_interface_atomic.cc ------------------------------------------===// 27ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 37ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// The LLVM Compiler Infrastructure 47ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 57ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// This file is distributed under the University of Illinois Open Source 67ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// License. See LICENSE.TXT for details. 77ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 87ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany//===----------------------------------------------------------------------===// 97ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 107ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// This file is a part of ThreadSanitizer (TSan), a race detector. 117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany//===----------------------------------------------------------------------===// 137ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 143c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov// ThreadSanitizer atomic operations are based on C++11/C1x standards. 152d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// For background see C++11 standard. A slightly older, publicly 163c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov// available draft of the standard (not entirely up-to-date, but close enough 173c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov// for casual browsing) is available here: 183c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf 193c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov// The following page contains more background information: 203c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov// http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/ 213c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov 2247b1634df012507799eb39aa17d4022d748ba67bAlexey Samsonov#include "sanitizer_common/sanitizer_placement_new.h" 23cafed29d27a9a1700aa3c828329f9db8e7592a6bDmitry Vyukov#include "sanitizer_common/sanitizer_stacktrace.h" 242d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#include "sanitizer_common/sanitizer_mutex.h" 257ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_flags.h" 267ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany#include "tsan_rtl.h" 277ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 287ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyusing namespace __tsan; // NOLINT 297ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 302d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// These should match declarations from public tsan_interface_atomic.h header. 312d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinestypedef unsigned char a8; 322d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinestypedef unsigned short a16; // NOLINT 332d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinestypedef unsigned int a32; 342d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinestypedef unsigned long long a64; // NOLINT 3586277eb844c4983c81de62d7c050e92fe7155788Stephen Hines#if !defined(SANITIZER_GO) && (defined(__SIZEOF_INT128__) \ 3686277eb844c4983c81de62d7c050e92fe7155788Stephen Hines || (__clang_major__ * 100 + __clang_minor__ >= 302)) && !defined(__mips64) 372d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines__extension__ typedef __int128 a128; 382d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines# define __TSAN_HAS_INT128 1 392d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#else 402d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines# define __TSAN_HAS_INT128 0 412d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#endif 422d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 4386277eb844c4983c81de62d7c050e92fe7155788Stephen Hines#if !defined(SANITIZER_GO) && __TSAN_HAS_INT128 442d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// Protects emulation of 128-bit atomic operations. 452d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstatic StaticSpinMutex mutex128; 466d1862363c88c183b0ed7740fca876342cf0474bStephen Hines#endif 472d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 482d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// Part of ABI, do not change. 492d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// http://llvm.org/viewvc/llvm-project/libcxx/trunk/include/atomic?view=markup 502d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinestypedef enum { 512d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines mo_relaxed, 522d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines mo_consume, 532d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines mo_acquire, 542d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines mo_release, 552d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines mo_acq_rel, 562d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines mo_seq_cst 572d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} morder; 58bb5aecb56d84e09106a88aca2d30fbb0b29b2e9cDmitry Vyukov 597ba89e5c1f7bd91a3c03d449b073c99c12f554ebDmitry Vyukovstatic bool IsLoadOrder(morder mo) { 607ba89e5c1f7bd91a3c03d449b073c99c12f554ebDmitry Vyukov return mo == mo_relaxed || mo == mo_consume 617ba89e5c1f7bd91a3c03d449b073c99c12f554ebDmitry Vyukov || mo == mo_acquire || mo == mo_seq_cst; 627ba89e5c1f7bd91a3c03d449b073c99c12f554ebDmitry Vyukov} 637ba89e5c1f7bd91a3c03d449b073c99c12f554ebDmitry Vyukov 647ba89e5c1f7bd91a3c03d449b073c99c12f554ebDmitry Vyukovstatic bool IsStoreOrder(morder mo) { 657ba89e5c1f7bd91a3c03d449b073c99c12f554ebDmitry Vyukov return mo == mo_relaxed || mo == mo_release || mo == mo_seq_cst; 667ba89e5c1f7bd91a3c03d449b073c99c12f554ebDmitry Vyukov} 677ba89e5c1f7bd91a3c03d449b073c99c12f554ebDmitry Vyukov 687ba89e5c1f7bd91a3c03d449b073c99c12f554ebDmitry Vyukovstatic bool IsReleaseOrder(morder mo) { 697ba89e5c1f7bd91a3c03d449b073c99c12f554ebDmitry Vyukov return mo == mo_release || mo == mo_acq_rel || mo == mo_seq_cst; 707ba89e5c1f7bd91a3c03d449b073c99c12f554ebDmitry Vyukov} 717ba89e5c1f7bd91a3c03d449b073c99c12f554ebDmitry Vyukov 727ba89e5c1f7bd91a3c03d449b073c99c12f554ebDmitry Vyukovstatic bool IsAcquireOrder(morder mo) { 737ba89e5c1f7bd91a3c03d449b073c99c12f554ebDmitry Vyukov return mo == mo_consume || mo == mo_acquire 747ba89e5c1f7bd91a3c03d449b073c99c12f554ebDmitry Vyukov || mo == mo_acq_rel || mo == mo_seq_cst; 757ba89e5c1f7bd91a3c03d449b073c99c12f554ebDmitry Vyukov} 767ba89e5c1f7bd91a3c03d449b073c99c12f554ebDmitry Vyukov 773c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukovstatic bool IsAcqRelOrder(morder mo) { 783c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov return mo == mo_acq_rel || mo == mo_seq_cst; 793c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov} 803c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov 81634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukovtemplate<typename T> T func_xchg(volatile T *v, T op) { 8237e6af28435fea112a68d7f1b86b99d24f967b04Dmitry Vyukov T res = __sync_lock_test_and_set(v, op); 8337e6af28435fea112a68d7f1b86b99d24f967b04Dmitry Vyukov // __sync_lock_test_and_set does not contain full barrier. 8437e6af28435fea112a68d7f1b86b99d24f967b04Dmitry Vyukov __sync_synchronize(); 8537e6af28435fea112a68d7f1b86b99d24f967b04Dmitry Vyukov return res; 863c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov} 873c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov 88634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukovtemplate<typename T> T func_add(volatile T *v, T op) { 89634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov return __sync_fetch_and_add(v, op); 903c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov} 913c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov 92634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukovtemplate<typename T> T func_sub(volatile T *v, T op) { 93634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov return __sync_fetch_and_sub(v, op); 943c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov} 953c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov 96634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukovtemplate<typename T> T func_and(volatile T *v, T op) { 97634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov return __sync_fetch_and_and(v, op); 983c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov} 993c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov 100634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukovtemplate<typename T> T func_or(volatile T *v, T op) { 101634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov return __sync_fetch_and_or(v, op); 1023c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov} 1033c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov 104634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukovtemplate<typename T> T func_xor(volatile T *v, T op) { 105634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov return __sync_fetch_and_xor(v, op); 1063c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov} 1073c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov 108634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukovtemplate<typename T> T func_nand(volatile T *v, T op) { 109634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov // clang does not support __sync_fetch_and_nand. 110634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov T cmp = *v; 111634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov for (;;) { 112634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov T newv = ~(cmp & op); 113634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov T cur = __sync_val_compare_and_swap(v, cmp, newv); 114634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov if (cmp == cur) 115634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov return cmp; 116634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov cmp = cur; 117634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov } 118634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov} 119634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov 120634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukovtemplate<typename T> T func_cas(volatile T *v, T cmp, T xch) { 121634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov return __sync_val_compare_and_swap(v, cmp, xch); 122634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov} 123634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov 124634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov// clang does not support 128-bit atomic ops. 125634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov// Atomic ops are executed under tsan internal mutex, 126634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov// here we assume that the atomic variables are not accessed 127634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov// from non-instrumented code. 12886277eb844c4983c81de62d7c050e92fe7155788Stephen Hines#if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && !defined(SANITIZER_GO) \ 12986277eb844c4983c81de62d7c050e92fe7155788Stephen Hines && __TSAN_HAS_INT128 130634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukova128 func_xchg(volatile a128 *v, a128 op) { 1312d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines SpinMutexLock lock(&mutex128); 132634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov a128 cmp = *v; 133634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov *v = op; 134634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov return cmp; 135634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov} 136634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov 137634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukova128 func_add(volatile a128 *v, a128 op) { 1382d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines SpinMutexLock lock(&mutex128); 139634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov a128 cmp = *v; 140634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov *v = cmp + op; 141634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov return cmp; 142634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov} 143634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov 144634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukova128 func_sub(volatile a128 *v, a128 op) { 1452d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines SpinMutexLock lock(&mutex128); 146634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov a128 cmp = *v; 147634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov *v = cmp - op; 148634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov return cmp; 149634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov} 150634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov 151634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukova128 func_and(volatile a128 *v, a128 op) { 1522d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines SpinMutexLock lock(&mutex128); 153634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov a128 cmp = *v; 154634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov *v = cmp & op; 155634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov return cmp; 156634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov} 157634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov 158634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukova128 func_or(volatile a128 *v, a128 op) { 1592d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines SpinMutexLock lock(&mutex128); 160634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov a128 cmp = *v; 161634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov *v = cmp | op; 162634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov return cmp; 1633c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov} 1643c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov 165634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukova128 func_xor(volatile a128 *v, a128 op) { 1662d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines SpinMutexLock lock(&mutex128); 167634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov a128 cmp = *v; 168634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov *v = cmp ^ op; 169634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov return cmp; 170634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov} 171634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov 172634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukova128 func_nand(volatile a128 *v, a128 op) { 1732d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines SpinMutexLock lock(&mutex128); 174634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov a128 cmp = *v; 175634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov *v = ~(cmp & op); 176634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov return cmp; 177634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov} 178634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov 179634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukova128 func_cas(volatile a128 *v, a128 cmp, a128 xch) { 1802d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines SpinMutexLock lock(&mutex128); 181634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov a128 cur = *v; 182634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov if (cur == cmp) 183634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov *v = xch; 184634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov return cur; 185634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov} 186634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov#endif 187634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov 1887ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanytemplate<typename T> 189334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukovstatic int SizeLog() { 190334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov if (sizeof(T) <= 1) 191334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov return kSizeLog1; 192334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov else if (sizeof(T) <= 2) 193334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov return kSizeLog2; 194334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov else if (sizeof(T) <= 4) 195334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov return kSizeLog4; 196334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov else 197334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov return kSizeLog8; 198334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov // For 16-byte atomics we also use 8-byte memory access, 199334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov // this leads to false negatives only in very obscure cases. 200334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov} 201334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov 20286277eb844c4983c81de62d7c050e92fe7155788Stephen Hines#ifndef SANITIZER_GO 2032d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstatic atomic_uint8_t *to_atomic(const volatile a8 *a) { 2046d1862363c88c183b0ed7740fca876342cf0474bStephen Hines return reinterpret_cast<atomic_uint8_t *>(const_cast<a8 *>(a)); 2052d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 2062d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 2072d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstatic atomic_uint16_t *to_atomic(const volatile a16 *a) { 2086d1862363c88c183b0ed7740fca876342cf0474bStephen Hines return reinterpret_cast<atomic_uint16_t *>(const_cast<a16 *>(a)); 2092d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 2106d1862363c88c183b0ed7740fca876342cf0474bStephen Hines#endif 2112d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 2122d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstatic atomic_uint32_t *to_atomic(const volatile a32 *a) { 2136d1862363c88c183b0ed7740fca876342cf0474bStephen Hines return reinterpret_cast<atomic_uint32_t *>(const_cast<a32 *>(a)); 2142d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 2152d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 2162d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstatic atomic_uint64_t *to_atomic(const volatile a64 *a) { 2176d1862363c88c183b0ed7740fca876342cf0474bStephen Hines return reinterpret_cast<atomic_uint64_t *>(const_cast<a64 *>(a)); 2182d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 2192d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 2202d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstatic memory_order to_mo(morder mo) { 2212d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines switch (mo) { 2222d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines case mo_relaxed: return memory_order_relaxed; 2232d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines case mo_consume: return memory_order_consume; 2242d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines case mo_acquire: return memory_order_acquire; 2252d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines case mo_release: return memory_order_release; 2262d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines case mo_acq_rel: return memory_order_acq_rel; 2272d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines case mo_seq_cst: return memory_order_seq_cst; 2282d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 2292d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines CHECK(0); 2302d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return memory_order_seq_cst; 2312d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 2322d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 2332d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinestemplate<typename T> 2342d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstatic T NoTsanAtomicLoad(const volatile T *a, morder mo) { 2352d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return atomic_load(to_atomic(a), to_mo(mo)); 2362d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 2372d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 23886277eb844c4983c81de62d7c050e92fe7155788Stephen Hines#if __TSAN_HAS_INT128 && !defined(SANITIZER_GO) 2392d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstatic a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) { 2402d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines SpinMutexLock lock(&mutex128); 2412d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return *a; 2422d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 2432d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#endif 2442d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 245334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukovtemplate<typename T> 2467ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystatic T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, 2477ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany morder mo) { 2487ba89e5c1f7bd91a3c03d449b073c99c12f554ebDmitry Vyukov CHECK(IsLoadOrder(mo)); 2493c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov // This fast-path is critical for performance. 2503c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov // Assume the access is atomic. 2512d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (!IsAcquireOrder(mo)) { 2523285866e45a8521c56ba6209daf3c9f91f844fd3Dmitry Vyukov MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>()); 2532d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return NoTsanAtomicLoad(a, mo); 254334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov } 2556a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, false); 256e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov AcquireImpl(thr, pc, &s->clock); 2572d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines T v = NoTsanAtomicLoad(a, mo); 2583c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov s->mtx.ReadUnlock(); 2593285866e45a8521c56ba6209daf3c9f91f844fd3Dmitry Vyukov MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>()); 2607ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return v; 2617ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 2627ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 2637ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanytemplate<typename T> 2642d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstatic void NoTsanAtomicStore(volatile T *a, T v, morder mo) { 2652d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines atomic_store(to_atomic(a), v, to_mo(mo)); 2662d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 2672d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 26886277eb844c4983c81de62d7c050e92fe7155788Stephen Hines#if __TSAN_HAS_INT128 && !defined(SANITIZER_GO) 2692d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstatic void NoTsanAtomicStore(volatile a128 *a, a128 v, morder mo) { 2702d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines SpinMutexLock lock(&mutex128); 2712d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines *a = v; 2722d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 2732d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#endif 2742d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 2752d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinestemplate<typename T> 2767ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystatic void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v, 2777ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany morder mo) { 2787ba89e5c1f7bd91a3c03d449b073c99c12f554ebDmitry Vyukov CHECK(IsStoreOrder(mo)); 2793285866e45a8521c56ba6209daf3c9f91f844fd3Dmitry Vyukov MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>()); 2803c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov // This fast-path is critical for performance. 2813c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov // Assume the access is atomic. 2823c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov // Strictly saying even relaxed store cuts off release sequence, 2833c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov // so must reset the clock. 2842d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (!IsReleaseOrder(mo)) { 2852d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines NoTsanAtomicStore(a, v, mo); 2863c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov return; 2873c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov } 288634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov __sync_synchronize(); 2896a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, true); 290e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov thr->fast_state.IncrementEpoch(); 291e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov // Can't increment epoch w/o writing to the trace as well. 292e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); 293e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov ReleaseImpl(thr, pc, &s->clock); 2942d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines NoTsanAtomicStore(a, v, mo); 2953c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov s->mtx.Unlock(); 2963c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov} 2973c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov 298634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukovtemplate<typename T, T (*F)(volatile T *v, T op)> 2993c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukovstatic T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { 3003285866e45a8521c56ba6209daf3c9f91f844fd3Dmitry Vyukov MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>()); 301210ce0ee0190170ca6fab6f5d953885ce5f9ac89Dmitry Vyukov SyncVar *s = 0; 302210ce0ee0190170ca6fab6f5d953885ce5f9ac89Dmitry Vyukov if (mo != mo_relaxed) { 3036a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, true); 304e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov thr->fast_state.IncrementEpoch(); 305e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov // Can't increment epoch w/o writing to the trace as well. 306e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); 307210ce0ee0190170ca6fab6f5d953885ce5f9ac89Dmitry Vyukov if (IsAcqRelOrder(mo)) 308e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov AcquireReleaseImpl(thr, pc, &s->clock); 309210ce0ee0190170ca6fab6f5d953885ce5f9ac89Dmitry Vyukov else if (IsReleaseOrder(mo)) 310e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov ReleaseImpl(thr, pc, &s->clock); 311210ce0ee0190170ca6fab6f5d953885ce5f9ac89Dmitry Vyukov else if (IsAcquireOrder(mo)) 312e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov AcquireImpl(thr, pc, &s->clock); 313210ce0ee0190170ca6fab6f5d953885ce5f9ac89Dmitry Vyukov } 314634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov v = F(a, v); 315210ce0ee0190170ca6fab6f5d953885ce5f9ac89Dmitry Vyukov if (s) 316210ce0ee0190170ca6fab6f5d953885ce5f9ac89Dmitry Vyukov s->mtx.Unlock(); 317634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov return v; 3187ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 3197ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3207ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanytemplate<typename T> 3212d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstatic T NoTsanAtomicExchange(volatile T *a, T v, morder mo) { 3222d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return func_xchg(a, v); 3232d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 3242d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 3252d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinestemplate<typename T> 3262d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstatic T NoTsanAtomicFetchAdd(volatile T *a, T v, morder mo) { 3272d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return func_add(a, v); 3282d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 3292d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 3302d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinestemplate<typename T> 3312d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstatic T NoTsanAtomicFetchSub(volatile T *a, T v, morder mo) { 3322d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return func_sub(a, v); 3332d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 3342d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 3352d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinestemplate<typename T> 3362d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstatic T NoTsanAtomicFetchAnd(volatile T *a, T v, morder mo) { 3372d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return func_and(a, v); 3382d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 3392d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 3402d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinestemplate<typename T> 3412d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstatic T NoTsanAtomicFetchOr(volatile T *a, T v, morder mo) { 3422d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return func_or(a, v); 3432d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 3442d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 3452d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinestemplate<typename T> 3462d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstatic T NoTsanAtomicFetchXor(volatile T *a, T v, morder mo) { 3472d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return func_xor(a, v); 3482d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 3492d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 3502d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinestemplate<typename T> 3512d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstatic T NoTsanAtomicFetchNand(volatile T *a, T v, morder mo) { 3522d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return func_nand(a, v); 3532d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 3542d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 3552d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinestemplate<typename T> 3567ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystatic T AtomicExchange(ThreadState *thr, uptr pc, volatile T *a, T v, 3577ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany morder mo) { 3583c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo); 3597ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 3607ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3617ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanytemplate<typename T> 3627ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystatic T AtomicFetchAdd(ThreadState *thr, uptr pc, volatile T *a, T v, 3637ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany morder mo) { 3643c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov return AtomicRMW<T, func_add>(thr, pc, a, v, mo); 3657ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 3667ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 3677ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanytemplate<typename T> 368529cfa05a7dd0cdbe97172045430243181e43522Dmitry Vyukovstatic T AtomicFetchSub(ThreadState *thr, uptr pc, volatile T *a, T v, 369529cfa05a7dd0cdbe97172045430243181e43522Dmitry Vyukov morder mo) { 3703c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov return AtomicRMW<T, func_sub>(thr, pc, a, v, mo); 371529cfa05a7dd0cdbe97172045430243181e43522Dmitry Vyukov} 372529cfa05a7dd0cdbe97172045430243181e43522Dmitry Vyukov 373529cfa05a7dd0cdbe97172045430243181e43522Dmitry Vyukovtemplate<typename T> 374cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukovstatic T AtomicFetchAnd(ThreadState *thr, uptr pc, volatile T *a, T v, 375cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov morder mo) { 3763c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov return AtomicRMW<T, func_and>(thr, pc, a, v, mo); 377cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov} 378cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov 379cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukovtemplate<typename T> 380cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukovstatic T AtomicFetchOr(ThreadState *thr, uptr pc, volatile T *a, T v, 381cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov morder mo) { 3823c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov return AtomicRMW<T, func_or>(thr, pc, a, v, mo); 383cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov} 384cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov 385cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukovtemplate<typename T> 386cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukovstatic T AtomicFetchXor(ThreadState *thr, uptr pc, volatile T *a, T v, 387cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov morder mo) { 3883c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov return AtomicRMW<T, func_xor>(thr, pc, a, v, mo); 389cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov} 390cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov 391cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukovtemplate<typename T> 39202b45d2aec154cb392cd6eb85e8dd16ed255351fDmitry Vyukovstatic T AtomicFetchNand(ThreadState *thr, uptr pc, volatile T *a, T v, 39302b45d2aec154cb392cd6eb85e8dd16ed255351fDmitry Vyukov morder mo) { 3943c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov return AtomicRMW<T, func_nand>(thr, pc, a, v, mo); 39502b45d2aec154cb392cd6eb85e8dd16ed255351fDmitry Vyukov} 39602b45d2aec154cb392cd6eb85e8dd16ed255351fDmitry Vyukov 39702b45d2aec154cb392cd6eb85e8dd16ed255351fDmitry Vyukovtemplate<typename T> 3982d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstatic bool NoTsanAtomicCAS(volatile T *a, T *c, T v, morder mo, morder fmo) { 3992d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return atomic_compare_exchange_strong(to_atomic(a), c, v, to_mo(mo)); 4002d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 4012d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 4022d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#if __TSAN_HAS_INT128 4032d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstatic bool NoTsanAtomicCAS(volatile a128 *a, a128 *c, a128 v, 4042d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines morder mo, morder fmo) { 4052d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines a128 old = *c; 4062d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines a128 cur = func_cas(a, old, v); 4072d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (cur == old) 4082d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return true; 4092d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines *c = cur; 4102d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return false; 4112d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 4122d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#endif 4132d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 4142d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinestemplate<typename T> 4156d1862363c88c183b0ed7740fca876342cf0474bStephen Hinesstatic T NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) { 4166d1862363c88c183b0ed7740fca876342cf0474bStephen Hines NoTsanAtomicCAS(a, &c, v, mo, fmo); 4176d1862363c88c183b0ed7740fca876342cf0474bStephen Hines return c; 4182d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 4192d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 4202d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinestemplate<typename T> 4217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystatic bool AtomicCAS(ThreadState *thr, uptr pc, 422fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov volatile T *a, T *c, T v, morder mo, morder fmo) { 4233c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov (void)fmo; // Unused because llvm does not pass it yet. 4243285866e45a8521c56ba6209daf3c9f91f844fd3Dmitry Vyukov MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>()); 425210ce0ee0190170ca6fab6f5d953885ce5f9ac89Dmitry Vyukov SyncVar *s = 0; 4262d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines bool write_lock = mo != mo_acquire && mo != mo_consume; 427210ce0ee0190170ca6fab6f5d953885ce5f9ac89Dmitry Vyukov if (mo != mo_relaxed) { 4286a211c5814e25d6745a5058cc0e499e5235d3821Stephen Hines s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, write_lock); 429e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov thr->fast_state.IncrementEpoch(); 430e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov // Can't increment epoch w/o writing to the trace as well. 431e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); 432210ce0ee0190170ca6fab6f5d953885ce5f9ac89Dmitry Vyukov if (IsAcqRelOrder(mo)) 433e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov AcquireReleaseImpl(thr, pc, &s->clock); 434210ce0ee0190170ca6fab6f5d953885ce5f9ac89Dmitry Vyukov else if (IsReleaseOrder(mo)) 435e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov ReleaseImpl(thr, pc, &s->clock); 436210ce0ee0190170ca6fab6f5d953885ce5f9ac89Dmitry Vyukov else if (IsAcquireOrder(mo)) 437e1ddbf9a458e81125a03fea721997565124294aeDmitry Vyukov AcquireImpl(thr, pc, &s->clock); 438210ce0ee0190170ca6fab6f5d953885ce5f9ac89Dmitry Vyukov } 439634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov T cc = *c; 440634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov T pr = func_cas(a, cc, v); 4412d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (s) { 4422d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (write_lock) 4432d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines s->mtx.Unlock(); 4442d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines else 4452d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines s->mtx.ReadUnlock(); 4462d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 447634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov if (pr == cc) 448634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov return true; 449634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov *c = pr; 450634f0476879e4509f8520f0f1542a27e0a10ccdaDmitry Vyukov return false; 4517ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 4527ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 453e337bf303dde9258bc7e2c87722a270fed96460bDmitry Vyukovtemplate<typename T> 454e337bf303dde9258bc7e2c87722a270fed96460bDmitry Vyukovstatic T AtomicCAS(ThreadState *thr, uptr pc, 455fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov volatile T *a, T c, T v, morder mo, morder fmo) { 456fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov AtomicCAS(thr, pc, a, &c, v, mo, fmo); 457e337bf303dde9258bc7e2c87722a270fed96460bDmitry Vyukov return c; 458e337bf303dde9258bc7e2c87722a270fed96460bDmitry Vyukov} 459e337bf303dde9258bc7e2c87722a270fed96460bDmitry Vyukov 46086277eb844c4983c81de62d7c050e92fe7155788Stephen Hines#ifndef SANITIZER_GO 4612d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesstatic void NoTsanAtomicFence(morder mo) { 4622d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __sync_synchronize(); 4632d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 4642d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 4657ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanystatic void AtomicFence(ThreadState *thr, uptr pc, morder mo) { 4663c0c408fe5268058c1347b51235d564cc081e575Dmitry Vyukov // FIXME(dvyukov): not implemented. 4677ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany __sync_synchronize(); 4687ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 4696d1862363c88c183b0ed7740fca876342cf0474bStephen Hines#endif 4706d1862363c88c183b0ed7740fca876342cf0474bStephen Hines 4716d1862363c88c183b0ed7740fca876342cf0474bStephen Hines// Interface functions follow. 47286277eb844c4983c81de62d7c050e92fe7155788Stephen Hines#ifndef SANITIZER_GO 4736d1862363c88c183b0ed7740fca876342cf0474bStephen Hines 4746d1862363c88c183b0ed7740fca876342cf0474bStephen Hines// C/C++ 4756d1862363c88c183b0ed7740fca876342cf0474bStephen Hines 4766d1862363c88c183b0ed7740fca876342cf0474bStephen Hines#define SCOPED_ATOMIC(func, ...) \ 4776d1862363c88c183b0ed7740fca876342cf0474bStephen Hines const uptr callpc = (uptr)__builtin_return_address(0); \ 4786d1862363c88c183b0ed7740fca876342cf0474bStephen Hines uptr pc = StackTrace::GetCurrentPc(); \ 4796d1862363c88c183b0ed7740fca876342cf0474bStephen Hines mo = flags()->force_seq_cst_atomics ? (morder)mo_seq_cst : mo; \ 4806d1862363c88c183b0ed7740fca876342cf0474bStephen Hines ThreadState *const thr = cur_thread(); \ 4816d1862363c88c183b0ed7740fca876342cf0474bStephen Hines if (thr->ignore_interceptors) \ 4826d1862363c88c183b0ed7740fca876342cf0474bStephen Hines return NoTsanAtomic##func(__VA_ARGS__); \ 4836d1862363c88c183b0ed7740fca876342cf0474bStephen Hines AtomicStatInc(thr, sizeof(*a), mo, StatAtomic##func); \ 4846d1862363c88c183b0ed7740fca876342cf0474bStephen Hines ScopedAtomic sa(thr, callpc, a, mo, __func__); \ 4856d1862363c88c183b0ed7740fca876342cf0474bStephen Hines return Atomic##func(thr, pc, __VA_ARGS__); \ 4866d1862363c88c183b0ed7740fca876342cf0474bStephen Hines/**/ 4876d1862363c88c183b0ed7740fca876342cf0474bStephen Hines 4886d1862363c88c183b0ed7740fca876342cf0474bStephen Hinesclass ScopedAtomic { 4896d1862363c88c183b0ed7740fca876342cf0474bStephen Hines public: 4906d1862363c88c183b0ed7740fca876342cf0474bStephen Hines ScopedAtomic(ThreadState *thr, uptr pc, const volatile void *a, 4916d1862363c88c183b0ed7740fca876342cf0474bStephen Hines morder mo, const char *func) 4926d1862363c88c183b0ed7740fca876342cf0474bStephen Hines : thr_(thr) { 4936d1862363c88c183b0ed7740fca876342cf0474bStephen Hines FuncEntry(thr_, pc); 4946d1862363c88c183b0ed7740fca876342cf0474bStephen Hines DPrintf("#%d: %s(%p, %d)\n", thr_->tid, func, a, mo); 4956d1862363c88c183b0ed7740fca876342cf0474bStephen Hines } 4966d1862363c88c183b0ed7740fca876342cf0474bStephen Hines ~ScopedAtomic() { 4976d1862363c88c183b0ed7740fca876342cf0474bStephen Hines ProcessPendingSignals(thr_); 4986d1862363c88c183b0ed7740fca876342cf0474bStephen Hines FuncExit(thr_); 4996d1862363c88c183b0ed7740fca876342cf0474bStephen Hines } 5006d1862363c88c183b0ed7740fca876342cf0474bStephen Hines private: 5016d1862363c88c183b0ed7740fca876342cf0474bStephen Hines ThreadState *thr_; 5026d1862363c88c183b0ed7740fca876342cf0474bStephen Hines}; 5036d1862363c88c183b0ed7740fca876342cf0474bStephen Hines 5046d1862363c88c183b0ed7740fca876342cf0474bStephen Hinesstatic void AtomicStatInc(ThreadState *thr, uptr size, morder mo, StatType t) { 5056d1862363c88c183b0ed7740fca876342cf0474bStephen Hines StatInc(thr, StatAtomic); 5066d1862363c88c183b0ed7740fca876342cf0474bStephen Hines StatInc(thr, t); 5076d1862363c88c183b0ed7740fca876342cf0474bStephen Hines StatInc(thr, size == 1 ? StatAtomic1 5086d1862363c88c183b0ed7740fca876342cf0474bStephen Hines : size == 2 ? StatAtomic2 5096d1862363c88c183b0ed7740fca876342cf0474bStephen Hines : size == 4 ? StatAtomic4 5106d1862363c88c183b0ed7740fca876342cf0474bStephen Hines : size == 8 ? StatAtomic8 5116d1862363c88c183b0ed7740fca876342cf0474bStephen Hines : StatAtomic16); 5126d1862363c88c183b0ed7740fca876342cf0474bStephen Hines StatInc(thr, mo == mo_relaxed ? StatAtomicRelaxed 5136d1862363c88c183b0ed7740fca876342cf0474bStephen Hines : mo == mo_consume ? StatAtomicConsume 5146d1862363c88c183b0ed7740fca876342cf0474bStephen Hines : mo == mo_acquire ? StatAtomicAcquire 5156d1862363c88c183b0ed7740fca876342cf0474bStephen Hines : mo == mo_release ? StatAtomicRelease 5166d1862363c88c183b0ed7740fca876342cf0474bStephen Hines : mo == mo_acq_rel ? StatAtomicAcq_Rel 5176d1862363c88c183b0ed7740fca876342cf0474bStephen Hines : StatAtomicSeq_Cst); 5186d1862363c88c183b0ed7740fca876342cf0474bStephen Hines} 5197ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 5202d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesextern "C" { 5212d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 5227ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanya8 __tsan_atomic8_load(const volatile a8 *a, morder mo) { 5237ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany SCOPED_ATOMIC(Load, a, mo); 5247ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 5257ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 5262d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 5277ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanya16 __tsan_atomic16_load(const volatile a16 *a, morder mo) { 5287ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany SCOPED_ATOMIC(Load, a, mo); 5297ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 5307ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 5312d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 5327ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanya32 __tsan_atomic32_load(const volatile a32 *a, morder mo) { 5337ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany SCOPED_ATOMIC(Load, a, mo); 5347ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 5357ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 5362d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 5377ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanya64 __tsan_atomic64_load(const volatile a64 *a, morder mo) { 5387ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany SCOPED_ATOMIC(Load, a, mo); 5397ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 5407ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 541c11e1e0f1401a95d0803317c4cdb83be3938146aDmitry Vyukov#if __TSAN_HAS_INT128 5422d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 54363da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukova128 __tsan_atomic128_load(const volatile a128 *a, morder mo) { 54463da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov SCOPED_ATOMIC(Load, a, mo); 54563da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov} 546c11e1e0f1401a95d0803317c4cdb83be3938146aDmitry Vyukov#endif 54763da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov 5482d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 5497ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) { 5507ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany SCOPED_ATOMIC(Store, a, v, mo); 5517ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 5527ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 5532d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 5547ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) { 5557ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany SCOPED_ATOMIC(Store, a, v, mo); 5567ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 5577ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 5582d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 5597ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) { 5607ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany SCOPED_ATOMIC(Store, a, v, mo); 5617ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 5627ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 5632d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 5647ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) { 5657ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany SCOPED_ATOMIC(Store, a, v, mo); 5667ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 5677ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 568c11e1e0f1401a95d0803317c4cdb83be3938146aDmitry Vyukov#if __TSAN_HAS_INT128 5692d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 57063da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukovvoid __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) { 57163da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov SCOPED_ATOMIC(Store, a, v, mo); 57263da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov} 573c11e1e0f1401a95d0803317c4cdb83be3938146aDmitry Vyukov#endif 57463da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov 5752d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 576cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukova8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) { 577cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov SCOPED_ATOMIC(Exchange, a, v, mo); 578cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov} 579cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov 5802d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 581cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukova16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) { 582cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov SCOPED_ATOMIC(Exchange, a, v, mo); 583cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov} 584cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov 5852d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 5867ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanya32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) { 5877ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany SCOPED_ATOMIC(Exchange, a, v, mo); 5887ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 5897ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 5902d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 5917ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanya64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) { 5927ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany SCOPED_ATOMIC(Exchange, a, v, mo); 5937ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 5947ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 595c11e1e0f1401a95d0803317c4cdb83be3938146aDmitry Vyukov#if __TSAN_HAS_INT128 5962d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 59763da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukova128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) { 59863da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov SCOPED_ATOMIC(Exchange, a, v, mo); 59963da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov} 600c11e1e0f1401a95d0803317c4cdb83be3938146aDmitry Vyukov#endif 60163da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov 6022d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 603cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukova8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) { 604cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov SCOPED_ATOMIC(FetchAdd, a, v, mo); 605cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov} 606cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov 6072d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 608cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukova16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) { 609cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov SCOPED_ATOMIC(FetchAdd, a, v, mo); 610cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov} 611cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov 6122d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 6137ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanya32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) { 6147ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany SCOPED_ATOMIC(FetchAdd, a, v, mo); 6157ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 6167ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 6172d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 6187ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanya64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) { 6197ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany SCOPED_ATOMIC(FetchAdd, a, v, mo); 6207ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 6217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 622c11e1e0f1401a95d0803317c4cdb83be3938146aDmitry Vyukov#if __TSAN_HAS_INT128 6232d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 62463da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukova128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) { 62563da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov SCOPED_ATOMIC(FetchAdd, a, v, mo); 62663da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov} 627c11e1e0f1401a95d0803317c4cdb83be3938146aDmitry Vyukov#endif 62863da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov 6292d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 630529cfa05a7dd0cdbe97172045430243181e43522Dmitry Vyukova8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) { 631529cfa05a7dd0cdbe97172045430243181e43522Dmitry Vyukov SCOPED_ATOMIC(FetchSub, a, v, mo); 632529cfa05a7dd0cdbe97172045430243181e43522Dmitry Vyukov} 633529cfa05a7dd0cdbe97172045430243181e43522Dmitry Vyukov 6342d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 635529cfa05a7dd0cdbe97172045430243181e43522Dmitry Vyukova16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) { 636529cfa05a7dd0cdbe97172045430243181e43522Dmitry Vyukov SCOPED_ATOMIC(FetchSub, a, v, mo); 637529cfa05a7dd0cdbe97172045430243181e43522Dmitry Vyukov} 638529cfa05a7dd0cdbe97172045430243181e43522Dmitry Vyukov 6392d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 640529cfa05a7dd0cdbe97172045430243181e43522Dmitry Vyukova32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) { 641529cfa05a7dd0cdbe97172045430243181e43522Dmitry Vyukov SCOPED_ATOMIC(FetchSub, a, v, mo); 642529cfa05a7dd0cdbe97172045430243181e43522Dmitry Vyukov} 643529cfa05a7dd0cdbe97172045430243181e43522Dmitry Vyukov 6442d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 645529cfa05a7dd0cdbe97172045430243181e43522Dmitry Vyukova64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) { 646529cfa05a7dd0cdbe97172045430243181e43522Dmitry Vyukov SCOPED_ATOMIC(FetchSub, a, v, mo); 647529cfa05a7dd0cdbe97172045430243181e43522Dmitry Vyukov} 648529cfa05a7dd0cdbe97172045430243181e43522Dmitry Vyukov 649c11e1e0f1401a95d0803317c4cdb83be3938146aDmitry Vyukov#if __TSAN_HAS_INT128 6502d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 65163da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukova128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) { 65263da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov SCOPED_ATOMIC(FetchSub, a, v, mo); 65363da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov} 654c11e1e0f1401a95d0803317c4cdb83be3938146aDmitry Vyukov#endif 65563da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov 6562d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 657cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukova8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) { 658cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov SCOPED_ATOMIC(FetchAnd, a, v, mo); 659cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov} 660cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov 6612d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 662cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukova16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) { 663cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov SCOPED_ATOMIC(FetchAnd, a, v, mo); 664cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov} 665cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov 6662d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 667cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukova32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) { 668cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov SCOPED_ATOMIC(FetchAnd, a, v, mo); 669cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov} 670cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov 6712d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 672cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukova64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) { 673cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov SCOPED_ATOMIC(FetchAnd, a, v, mo); 674cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov} 675cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov 676c11e1e0f1401a95d0803317c4cdb83be3938146aDmitry Vyukov#if __TSAN_HAS_INT128 6772d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 67863da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukova128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) { 67963da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov SCOPED_ATOMIC(FetchAnd, a, v, mo); 68063da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov} 681c11e1e0f1401a95d0803317c4cdb83be3938146aDmitry Vyukov#endif 68263da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov 6832d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 684cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukova8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) { 685cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov SCOPED_ATOMIC(FetchOr, a, v, mo); 686cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov} 687cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov 6882d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 689cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukova16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) { 690cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov SCOPED_ATOMIC(FetchOr, a, v, mo); 691cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov} 692cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov 6932d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 694cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukova32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) { 695cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov SCOPED_ATOMIC(FetchOr, a, v, mo); 696cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov} 697cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov 6982d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 699cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukova64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) { 700cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov SCOPED_ATOMIC(FetchOr, a, v, mo); 701cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov} 702cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov 703c11e1e0f1401a95d0803317c4cdb83be3938146aDmitry Vyukov#if __TSAN_HAS_INT128 7042d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 70563da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukova128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) { 70663da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov SCOPED_ATOMIC(FetchOr, a, v, mo); 70763da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov} 708c11e1e0f1401a95d0803317c4cdb83be3938146aDmitry Vyukov#endif 70963da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov 7102d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 711cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukova8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) { 712cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov SCOPED_ATOMIC(FetchXor, a, v, mo); 713cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov} 714cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov 7152d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 716cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukova16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) { 717cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov SCOPED_ATOMIC(FetchXor, a, v, mo); 718cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov} 719cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov 7202d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 721cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukova32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) { 722cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov SCOPED_ATOMIC(FetchXor, a, v, mo); 723cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov} 724cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov 7252d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 726cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukova64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) { 727cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov SCOPED_ATOMIC(FetchXor, a, v, mo); 728cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov} 729cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov 730c11e1e0f1401a95d0803317c4cdb83be3938146aDmitry Vyukov#if __TSAN_HAS_INT128 7312d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 73263da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukova128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) { 73363da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov SCOPED_ATOMIC(FetchXor, a, v, mo); 73463da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov} 735c11e1e0f1401a95d0803317c4cdb83be3938146aDmitry Vyukov#endif 73663da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov 7372d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 73802b45d2aec154cb392cd6eb85e8dd16ed255351fDmitry Vyukova8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) { 73902b45d2aec154cb392cd6eb85e8dd16ed255351fDmitry Vyukov SCOPED_ATOMIC(FetchNand, a, v, mo); 74002b45d2aec154cb392cd6eb85e8dd16ed255351fDmitry Vyukov} 74102b45d2aec154cb392cd6eb85e8dd16ed255351fDmitry Vyukov 7422d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 74302b45d2aec154cb392cd6eb85e8dd16ed255351fDmitry Vyukova16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) { 74402b45d2aec154cb392cd6eb85e8dd16ed255351fDmitry Vyukov SCOPED_ATOMIC(FetchNand, a, v, mo); 74502b45d2aec154cb392cd6eb85e8dd16ed255351fDmitry Vyukov} 74602b45d2aec154cb392cd6eb85e8dd16ed255351fDmitry Vyukov 7472d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 74802b45d2aec154cb392cd6eb85e8dd16ed255351fDmitry Vyukova32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) { 74902b45d2aec154cb392cd6eb85e8dd16ed255351fDmitry Vyukov SCOPED_ATOMIC(FetchNand, a, v, mo); 75002b45d2aec154cb392cd6eb85e8dd16ed255351fDmitry Vyukov} 75102b45d2aec154cb392cd6eb85e8dd16ed255351fDmitry Vyukov 7522d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 75302b45d2aec154cb392cd6eb85e8dd16ed255351fDmitry Vyukova64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) { 75402b45d2aec154cb392cd6eb85e8dd16ed255351fDmitry Vyukov SCOPED_ATOMIC(FetchNand, a, v, mo); 75502b45d2aec154cb392cd6eb85e8dd16ed255351fDmitry Vyukov} 75602b45d2aec154cb392cd6eb85e8dd16ed255351fDmitry Vyukov 757c11e1e0f1401a95d0803317c4cdb83be3938146aDmitry Vyukov#if __TSAN_HAS_INT128 7582d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 75963da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukova128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) { 76063da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov SCOPED_ATOMIC(FetchNand, a, v, mo); 76163da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov} 762c11e1e0f1401a95d0803317c4cdb83be3938146aDmitry Vyukov#endif 76363da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov 7642d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 765cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukovint __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v, 766fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov morder mo, morder fmo) { 767fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 768cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov} 769cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov 7702d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 771cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukovint __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v, 772fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov morder mo, morder fmo) { 773fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 774cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov} 775cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov 7762d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 7777ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyint __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v, 778fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov morder mo, morder fmo) { 779fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 7807ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 7817ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 7822d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 7837ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyint __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v, 784fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov morder mo, morder fmo) { 785fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 7867ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 7877ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 788c11e1e0f1401a95d0803317c4cdb83be3938146aDmitry Vyukov#if __TSAN_HAS_INT128 7892d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 79063da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukovint __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v, 79163da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov morder mo, morder fmo) { 79263da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 79363da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov} 794c11e1e0f1401a95d0803317c4cdb83be3938146aDmitry Vyukov#endif 79563da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov 7962d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 797cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukovint __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v, 798fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov morder mo, morder fmo) { 799fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 800cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov} 801cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov 8022d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 803cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukovint __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v, 804fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov morder mo, morder fmo) { 805fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 806cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov} 807cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov 8082d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 809cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukovint __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v, 810fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov morder mo, morder fmo) { 811fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 812cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov} 813cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov 8142d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 815cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukovint __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v, 816fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov morder mo, morder fmo) { 817fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 818cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov} 819cdfb33a69a0c96c6912036ad85224972db3a7e7dDmitry Vyukov 820c11e1e0f1401a95d0803317c4cdb83be3938146aDmitry Vyukov#if __TSAN_HAS_INT128 8212d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 82263da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukovint __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v, 82363da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov morder mo, morder fmo) { 82463da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 82563da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov} 826c11e1e0f1401a95d0803317c4cdb83be3938146aDmitry Vyukov#endif 82763da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov 8282d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 829e337bf303dde9258bc7e2c87722a270fed96460bDmitry Vyukova8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v, 830fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov morder mo, morder fmo) { 831fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 832e337bf303dde9258bc7e2c87722a270fed96460bDmitry Vyukov} 8332d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 8342d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 835e337bf303dde9258bc7e2c87722a270fed96460bDmitry Vyukova16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v, 836fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov morder mo, morder fmo) { 837fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 838e337bf303dde9258bc7e2c87722a270fed96460bDmitry Vyukov} 839e337bf303dde9258bc7e2c87722a270fed96460bDmitry Vyukov 8402d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 841e337bf303dde9258bc7e2c87722a270fed96460bDmitry Vyukova32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v, 842fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov morder mo, morder fmo) { 843fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 844e337bf303dde9258bc7e2c87722a270fed96460bDmitry Vyukov} 845e337bf303dde9258bc7e2c87722a270fed96460bDmitry Vyukov 8462d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 847e337bf303dde9258bc7e2c87722a270fed96460bDmitry Vyukova64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v, 848fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov morder mo, morder fmo) { 849fc575e52636c803262fc24bacf63a95f731b1314Dmitry Vyukov SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 850e337bf303dde9258bc7e2c87722a270fed96460bDmitry Vyukov} 851e337bf303dde9258bc7e2c87722a270fed96460bDmitry Vyukov 852c11e1e0f1401a95d0803317c4cdb83be3938146aDmitry Vyukov#if __TSAN_HAS_INT128 8532d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 854f2c29a5dc56feed66e81a7fb3f03ef2c5ea76870Richard Smitha128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v, 85563da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov morder mo, morder fmo) { 85663da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); 85763da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov} 858c11e1e0f1401a95d0803317c4cdb83be3938146aDmitry Vyukov#endif 85963da509a995a3cac9bf5495b5cb94c527ff96ce1Dmitry Vyukov 8602d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 8617ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanyvoid __tsan_atomic_thread_fence(morder mo) { 862bb5aecb56d84e09106a88aca2d30fbb0b29b2e9cDmitry Vyukov char* a = 0; 8637ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany SCOPED_ATOMIC(Fence, mo); 8647ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} 865529cfa05a7dd0cdbe97172045430243181e43522Dmitry Vyukov 8662d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesSANITIZER_INTERFACE_ATTRIBUTE 867529cfa05a7dd0cdbe97172045430243181e43522Dmitry Vyukovvoid __tsan_atomic_signal_fence(morder mo) { 868529cfa05a7dd0cdbe97172045430243181e43522Dmitry Vyukov} 8692d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} // extern "C" 8706d1862363c88c183b0ed7740fca876342cf0474bStephen Hines 87186277eb844c4983c81de62d7c050e92fe7155788Stephen Hines#else // #ifndef SANITIZER_GO 8726d1862363c88c183b0ed7740fca876342cf0474bStephen Hines 8736d1862363c88c183b0ed7740fca876342cf0474bStephen Hines// Go 8746d1862363c88c183b0ed7740fca876342cf0474bStephen Hines 8756d1862363c88c183b0ed7740fca876342cf0474bStephen Hines#define ATOMIC(func, ...) \ 8766d1862363c88c183b0ed7740fca876342cf0474bStephen Hines if (thr->ignore_sync) { \ 8776d1862363c88c183b0ed7740fca876342cf0474bStephen Hines NoTsanAtomic##func(__VA_ARGS__); \ 8786d1862363c88c183b0ed7740fca876342cf0474bStephen Hines } else { \ 8796d1862363c88c183b0ed7740fca876342cf0474bStephen Hines FuncEntry(thr, cpc); \ 8806d1862363c88c183b0ed7740fca876342cf0474bStephen Hines Atomic##func(thr, pc, __VA_ARGS__); \ 8816d1862363c88c183b0ed7740fca876342cf0474bStephen Hines FuncExit(thr); \ 8826d1862363c88c183b0ed7740fca876342cf0474bStephen Hines } \ 8836d1862363c88c183b0ed7740fca876342cf0474bStephen Hines/**/ 8846d1862363c88c183b0ed7740fca876342cf0474bStephen Hines 8856d1862363c88c183b0ed7740fca876342cf0474bStephen Hines#define ATOMIC_RET(func, ret, ...) \ 8866d1862363c88c183b0ed7740fca876342cf0474bStephen Hines if (thr->ignore_sync) { \ 8876d1862363c88c183b0ed7740fca876342cf0474bStephen Hines (ret) = NoTsanAtomic##func(__VA_ARGS__); \ 8886d1862363c88c183b0ed7740fca876342cf0474bStephen Hines } else { \ 8896d1862363c88c183b0ed7740fca876342cf0474bStephen Hines FuncEntry(thr, cpc); \ 8906d1862363c88c183b0ed7740fca876342cf0474bStephen Hines (ret) = Atomic##func(thr, pc, __VA_ARGS__); \ 8916d1862363c88c183b0ed7740fca876342cf0474bStephen Hines FuncExit(thr); \ 8926d1862363c88c183b0ed7740fca876342cf0474bStephen Hines } \ 8936d1862363c88c183b0ed7740fca876342cf0474bStephen Hines/**/ 8946d1862363c88c183b0ed7740fca876342cf0474bStephen Hines 8956d1862363c88c183b0ed7740fca876342cf0474bStephen Hinesextern "C" { 8966d1862363c88c183b0ed7740fca876342cf0474bStephen HinesSANITIZER_INTERFACE_ATTRIBUTE 8976d1862363c88c183b0ed7740fca876342cf0474bStephen Hinesvoid __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { 8986d1862363c88c183b0ed7740fca876342cf0474bStephen Hines ATOMIC_RET(Load, *(a32*)(a+8), *(a32**)a, mo_acquire); 8996d1862363c88c183b0ed7740fca876342cf0474bStephen Hines} 9006d1862363c88c183b0ed7740fca876342cf0474bStephen Hines 9016d1862363c88c183b0ed7740fca876342cf0474bStephen HinesSANITIZER_INTERFACE_ATTRIBUTE 9026d1862363c88c183b0ed7740fca876342cf0474bStephen Hinesvoid __tsan_go_atomic64_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { 9036d1862363c88c183b0ed7740fca876342cf0474bStephen Hines ATOMIC_RET(Load, *(a64*)(a+8), *(a64**)a, mo_acquire); 9046d1862363c88c183b0ed7740fca876342cf0474bStephen Hines} 9056d1862363c88c183b0ed7740fca876342cf0474bStephen Hines 9066d1862363c88c183b0ed7740fca876342cf0474bStephen HinesSANITIZER_INTERFACE_ATTRIBUTE 9076d1862363c88c183b0ed7740fca876342cf0474bStephen Hinesvoid __tsan_go_atomic32_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { 9086d1862363c88c183b0ed7740fca876342cf0474bStephen Hines ATOMIC(Store, *(a32**)a, *(a32*)(a+8), mo_release); 9096d1862363c88c183b0ed7740fca876342cf0474bStephen Hines} 9106d1862363c88c183b0ed7740fca876342cf0474bStephen Hines 9116d1862363c88c183b0ed7740fca876342cf0474bStephen HinesSANITIZER_INTERFACE_ATTRIBUTE 9126d1862363c88c183b0ed7740fca876342cf0474bStephen Hinesvoid __tsan_go_atomic64_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { 9136d1862363c88c183b0ed7740fca876342cf0474bStephen Hines ATOMIC(Store, *(a64**)a, *(a64*)(a+8), mo_release); 9146d1862363c88c183b0ed7740fca876342cf0474bStephen Hines} 9156d1862363c88c183b0ed7740fca876342cf0474bStephen Hines 9166d1862363c88c183b0ed7740fca876342cf0474bStephen HinesSANITIZER_INTERFACE_ATTRIBUTE 9176d1862363c88c183b0ed7740fca876342cf0474bStephen Hinesvoid __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { 9186d1862363c88c183b0ed7740fca876342cf0474bStephen Hines ATOMIC_RET(FetchAdd, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel); 9196d1862363c88c183b0ed7740fca876342cf0474bStephen Hines} 9206d1862363c88c183b0ed7740fca876342cf0474bStephen Hines 9216d1862363c88c183b0ed7740fca876342cf0474bStephen HinesSANITIZER_INTERFACE_ATTRIBUTE 9226d1862363c88c183b0ed7740fca876342cf0474bStephen Hinesvoid __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { 9236d1862363c88c183b0ed7740fca876342cf0474bStephen Hines ATOMIC_RET(FetchAdd, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel); 9246d1862363c88c183b0ed7740fca876342cf0474bStephen Hines} 9256d1862363c88c183b0ed7740fca876342cf0474bStephen Hines 9266d1862363c88c183b0ed7740fca876342cf0474bStephen HinesSANITIZER_INTERFACE_ATTRIBUTE 9276d1862363c88c183b0ed7740fca876342cf0474bStephen Hinesvoid __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { 9286d1862363c88c183b0ed7740fca876342cf0474bStephen Hines ATOMIC_RET(Exchange, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel); 9296d1862363c88c183b0ed7740fca876342cf0474bStephen Hines} 9306d1862363c88c183b0ed7740fca876342cf0474bStephen Hines 9316d1862363c88c183b0ed7740fca876342cf0474bStephen HinesSANITIZER_INTERFACE_ATTRIBUTE 9326d1862363c88c183b0ed7740fca876342cf0474bStephen Hinesvoid __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { 9336d1862363c88c183b0ed7740fca876342cf0474bStephen Hines ATOMIC_RET(Exchange, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel); 9346d1862363c88c183b0ed7740fca876342cf0474bStephen Hines} 9356d1862363c88c183b0ed7740fca876342cf0474bStephen Hines 9366d1862363c88c183b0ed7740fca876342cf0474bStephen HinesSANITIZER_INTERFACE_ATTRIBUTE 9376d1862363c88c183b0ed7740fca876342cf0474bStephen Hinesvoid __tsan_go_atomic32_compare_exchange( 9386d1862363c88c183b0ed7740fca876342cf0474bStephen Hines ThreadState *thr, uptr cpc, uptr pc, u8 *a) { 9396d1862363c88c183b0ed7740fca876342cf0474bStephen Hines a32 cur = 0; 9406d1862363c88c183b0ed7740fca876342cf0474bStephen Hines a32 cmp = *(a32*)(a+8); 9416d1862363c88c183b0ed7740fca876342cf0474bStephen Hines ATOMIC_RET(CAS, cur, *(a32**)a, cmp, *(a32*)(a+12), mo_acq_rel, mo_acquire); 9426d1862363c88c183b0ed7740fca876342cf0474bStephen Hines *(bool*)(a+16) = (cur == cmp); 9436d1862363c88c183b0ed7740fca876342cf0474bStephen Hines} 9446d1862363c88c183b0ed7740fca876342cf0474bStephen Hines 9456d1862363c88c183b0ed7740fca876342cf0474bStephen HinesSANITIZER_INTERFACE_ATTRIBUTE 9466d1862363c88c183b0ed7740fca876342cf0474bStephen Hinesvoid __tsan_go_atomic64_compare_exchange( 9476d1862363c88c183b0ed7740fca876342cf0474bStephen Hines ThreadState *thr, uptr cpc, uptr pc, u8 *a) { 9486d1862363c88c183b0ed7740fca876342cf0474bStephen Hines a64 cur = 0; 9496d1862363c88c183b0ed7740fca876342cf0474bStephen Hines a64 cmp = *(a64*)(a+8); 9506d1862363c88c183b0ed7740fca876342cf0474bStephen Hines ATOMIC_RET(CAS, cur, *(a64**)a, cmp, *(a64*)(a+16), mo_acq_rel, mo_acquire); 9516d1862363c88c183b0ed7740fca876342cf0474bStephen Hines *(bool*)(a+24) = (cur == cmp); 9526d1862363c88c183b0ed7740fca876342cf0474bStephen Hines} 9536d1862363c88c183b0ed7740fca876342cf0474bStephen Hines} // extern "C" 95486277eb844c4983c81de62d7c050e92fe7155788Stephen Hines#endif // #ifndef SANITIZER_GO 955