1//===-- tsan_interface_atomic.cc ------------------------------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file is a part of ThreadSanitizer (TSan), a race detector.
11//
12//===----------------------------------------------------------------------===//
13
14// ThreadSanitizer atomic operations are based on C++11/C1x standards.
15// For background see C++11 standard.  A slightly older, publicly
16// available draft of the standard (not entirely up-to-date, but close enough
17// for casual browsing) is available here:
18// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf
19// The following page contains more background information:
20// http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/
21
22#include "sanitizer_common/sanitizer_placement_new.h"
23#include "sanitizer_common/sanitizer_stacktrace.h"
24#include "sanitizer_common/sanitizer_mutex.h"
25#include "tsan_flags.h"
26#include "tsan_rtl.h"
27
28using namespace __tsan;  // NOLINT
29
30#define SCOPED_ATOMIC(func, ...) \
31    const uptr callpc = (uptr)__builtin_return_address(0); \
32    uptr pc = __sanitizer::StackTrace::GetCurrentPc(); \
33    mo = flags()->force_seq_cst_atomics ? (morder)mo_seq_cst : mo; \
34    ThreadState *const thr = cur_thread(); \
35    if (thr->ignore_interceptors) \
36      return NoTsanAtomic##func(__VA_ARGS__); \
37    AtomicStatInc(thr, sizeof(*a), mo, StatAtomic##func); \
38    ScopedAtomic sa(thr, callpc, a, mo, __func__); \
39    return Atomic##func(thr, pc, __VA_ARGS__); \
40/**/
41
42// These should match declarations from public tsan_interface_atomic.h header.
43typedef unsigned char      a8;
44typedef unsigned short     a16;  // NOLINT
45typedef unsigned int       a32;
46typedef unsigned long long a64;  // NOLINT
47#if defined(__SIZEOF_INT128__) \
48    || (__clang_major__ * 100 + __clang_minor__ >= 302)
49__extension__ typedef __int128 a128;
50# define __TSAN_HAS_INT128 1
51#else
52# define __TSAN_HAS_INT128 0
53#endif
54
55// Protects emulation of 128-bit atomic operations.
56static StaticSpinMutex mutex128;
57
58// Part of ABI, do not change.
59// http://llvm.org/viewvc/llvm-project/libcxx/trunk/include/atomic?view=markup
60typedef enum {
61  mo_relaxed,
62  mo_consume,
63  mo_acquire,
64  mo_release,
65  mo_acq_rel,
66  mo_seq_cst
67} morder;
68
69class ScopedAtomic {
70 public:
71  ScopedAtomic(ThreadState *thr, uptr pc, const volatile void *a,
72               morder mo, const char *func)
73      : thr_(thr) {
74    FuncEntry(thr_, pc);
75    DPrintf("#%d: %s(%p, %d)\n", thr_->tid, func, a, mo);
76  }
77  ~ScopedAtomic() {
78    ProcessPendingSignals(thr_);
79    FuncExit(thr_);
80  }
81 private:
82  ThreadState *thr_;
83};
84
85static void AtomicStatInc(ThreadState *thr, uptr size, morder mo, StatType t) {
86  StatInc(thr, StatAtomic);
87  StatInc(thr, t);
88  StatInc(thr, size == 1 ? StatAtomic1
89             : size == 2 ? StatAtomic2
90             : size == 4 ? StatAtomic4
91             : size == 8 ? StatAtomic8
92             :             StatAtomic16);
93  StatInc(thr, mo == mo_relaxed ? StatAtomicRelaxed
94             : mo == mo_consume ? StatAtomicConsume
95             : mo == mo_acquire ? StatAtomicAcquire
96             : mo == mo_release ? StatAtomicRelease
97             : mo == mo_acq_rel ? StatAtomicAcq_Rel
98             :                    StatAtomicSeq_Cst);
99}
100
101static bool IsLoadOrder(morder mo) {
102  return mo == mo_relaxed || mo == mo_consume
103      || mo == mo_acquire || mo == mo_seq_cst;
104}
105
106static bool IsStoreOrder(morder mo) {
107  return mo == mo_relaxed || mo == mo_release || mo == mo_seq_cst;
108}
109
110static bool IsReleaseOrder(morder mo) {
111  return mo == mo_release || mo == mo_acq_rel || mo == mo_seq_cst;
112}
113
114static bool IsAcquireOrder(morder mo) {
115  return mo == mo_consume || mo == mo_acquire
116      || mo == mo_acq_rel || mo == mo_seq_cst;
117}
118
119static bool IsAcqRelOrder(morder mo) {
120  return mo == mo_acq_rel || mo == mo_seq_cst;
121}
122
123template<typename T> T func_xchg(volatile T *v, T op) {
124  T res = __sync_lock_test_and_set(v, op);
125  // __sync_lock_test_and_set does not contain full barrier.
126  __sync_synchronize();
127  return res;
128}
129
130template<typename T> T func_add(volatile T *v, T op) {
131  return __sync_fetch_and_add(v, op);
132}
133
134template<typename T> T func_sub(volatile T *v, T op) {
135  return __sync_fetch_and_sub(v, op);
136}
137
138template<typename T> T func_and(volatile T *v, T op) {
139  return __sync_fetch_and_and(v, op);
140}
141
142template<typename T> T func_or(volatile T *v, T op) {
143  return __sync_fetch_and_or(v, op);
144}
145
146template<typename T> T func_xor(volatile T *v, T op) {
147  return __sync_fetch_and_xor(v, op);
148}
149
150template<typename T> T func_nand(volatile T *v, T op) {
151  // clang does not support __sync_fetch_and_nand.
152  T cmp = *v;
153  for (;;) {
154    T newv = ~(cmp & op);
155    T cur = __sync_val_compare_and_swap(v, cmp, newv);
156    if (cmp == cur)
157      return cmp;
158    cmp = cur;
159  }
160}
161
162template<typename T> T func_cas(volatile T *v, T cmp, T xch) {
163  return __sync_val_compare_and_swap(v, cmp, xch);
164}
165
166// clang does not support 128-bit atomic ops.
167// Atomic ops are executed under tsan internal mutex,
168// here we assume that the atomic variables are not accessed
169// from non-instrumented code.
170#ifndef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16
171a128 func_xchg(volatile a128 *v, a128 op) {
172  SpinMutexLock lock(&mutex128);
173  a128 cmp = *v;
174  *v = op;
175  return cmp;
176}
177
178a128 func_add(volatile a128 *v, a128 op) {
179  SpinMutexLock lock(&mutex128);
180  a128 cmp = *v;
181  *v = cmp + op;
182  return cmp;
183}
184
185a128 func_sub(volatile a128 *v, a128 op) {
186  SpinMutexLock lock(&mutex128);
187  a128 cmp = *v;
188  *v = cmp - op;
189  return cmp;
190}
191
192a128 func_and(volatile a128 *v, a128 op) {
193  SpinMutexLock lock(&mutex128);
194  a128 cmp = *v;
195  *v = cmp & op;
196  return cmp;
197}
198
199a128 func_or(volatile a128 *v, a128 op) {
200  SpinMutexLock lock(&mutex128);
201  a128 cmp = *v;
202  *v = cmp | op;
203  return cmp;
204}
205
206a128 func_xor(volatile a128 *v, a128 op) {
207  SpinMutexLock lock(&mutex128);
208  a128 cmp = *v;
209  *v = cmp ^ op;
210  return cmp;
211}
212
213a128 func_nand(volatile a128 *v, a128 op) {
214  SpinMutexLock lock(&mutex128);
215  a128 cmp = *v;
216  *v = ~(cmp & op);
217  return cmp;
218}
219
220a128 func_cas(volatile a128 *v, a128 cmp, a128 xch) {
221  SpinMutexLock lock(&mutex128);
222  a128 cur = *v;
223  if (cur == cmp)
224    *v = xch;
225  return cur;
226}
227#endif
228
229template<typename T>
230static int SizeLog() {
231  if (sizeof(T) <= 1)
232    return kSizeLog1;
233  else if (sizeof(T) <= 2)
234    return kSizeLog2;
235  else if (sizeof(T) <= 4)
236    return kSizeLog4;
237  else
238    return kSizeLog8;
239  // For 16-byte atomics we also use 8-byte memory access,
240  // this leads to false negatives only in very obscure cases.
241}
242
243static atomic_uint8_t *to_atomic(const volatile a8 *a) {
244  return (atomic_uint8_t*)a;
245}
246
247static atomic_uint16_t *to_atomic(const volatile a16 *a) {
248  return (atomic_uint16_t*)a;
249}
250
251static atomic_uint32_t *to_atomic(const volatile a32 *a) {
252  return (atomic_uint32_t*)a;
253}
254
255static atomic_uint64_t *to_atomic(const volatile a64 *a) {
256  return (atomic_uint64_t*)a;
257}
258
259static memory_order to_mo(morder mo) {
260  switch (mo) {
261  case mo_relaxed: return memory_order_relaxed;
262  case mo_consume: return memory_order_consume;
263  case mo_acquire: return memory_order_acquire;
264  case mo_release: return memory_order_release;
265  case mo_acq_rel: return memory_order_acq_rel;
266  case mo_seq_cst: return memory_order_seq_cst;
267  }
268  CHECK(0);
269  return memory_order_seq_cst;
270}
271
272template<typename T>
273static T NoTsanAtomicLoad(const volatile T *a, morder mo) {
274  return atomic_load(to_atomic(a), to_mo(mo));
275}
276
277#if __TSAN_HAS_INT128
278static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) {
279  SpinMutexLock lock(&mutex128);
280  return *a;
281}
282#endif
283
284template<typename T>
285static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a,
286    morder mo) {
287  CHECK(IsLoadOrder(mo));
288  // This fast-path is critical for performance.
289  // Assume the access is atomic.
290  if (!IsAcquireOrder(mo)) {
291    MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());
292    return NoTsanAtomicLoad(a, mo);
293  }
294  SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, false);
295  AcquireImpl(thr, pc, &s->clock);
296  T v = NoTsanAtomicLoad(a, mo);
297  s->mtx.ReadUnlock();
298  MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());
299  return v;
300}
301
302template<typename T>
303static void NoTsanAtomicStore(volatile T *a, T v, morder mo) {
304  atomic_store(to_atomic(a), v, to_mo(mo));
305}
306
307#if __TSAN_HAS_INT128
308static void NoTsanAtomicStore(volatile a128 *a, a128 v, morder mo) {
309  SpinMutexLock lock(&mutex128);
310  *a = v;
311}
312#endif
313
314template<typename T>
315static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v,
316    morder mo) {
317  CHECK(IsStoreOrder(mo));
318  MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
319  // This fast-path is critical for performance.
320  // Assume the access is atomic.
321  // Strictly saying even relaxed store cuts off release sequence,
322  // so must reset the clock.
323  if (!IsReleaseOrder(mo)) {
324    NoTsanAtomicStore(a, v, mo);
325    return;
326  }
327  __sync_synchronize();
328  SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, true);
329  thr->fast_state.IncrementEpoch();
330  // Can't increment epoch w/o writing to the trace as well.
331  TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
332  ReleaseImpl(thr, pc, &s->clock);
333  NoTsanAtomicStore(a, v, mo);
334  s->mtx.Unlock();
335}
336
337template<typename T, T (*F)(volatile T *v, T op)>
338static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) {
339  MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
340  SyncVar *s = 0;
341  if (mo != mo_relaxed) {
342    s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, true);
343    thr->fast_state.IncrementEpoch();
344    // Can't increment epoch w/o writing to the trace as well.
345    TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
346    if (IsAcqRelOrder(mo))
347      AcquireReleaseImpl(thr, pc, &s->clock);
348    else if (IsReleaseOrder(mo))
349      ReleaseImpl(thr, pc, &s->clock);
350    else if (IsAcquireOrder(mo))
351      AcquireImpl(thr, pc, &s->clock);
352  }
353  v = F(a, v);
354  if (s)
355    s->mtx.Unlock();
356  return v;
357}
358
359template<typename T>
360static T NoTsanAtomicExchange(volatile T *a, T v, morder mo) {
361  return func_xchg(a, v);
362}
363
364template<typename T>
365static T NoTsanAtomicFetchAdd(volatile T *a, T v, morder mo) {
366  return func_add(a, v);
367}
368
369template<typename T>
370static T NoTsanAtomicFetchSub(volatile T *a, T v, morder mo) {
371  return func_sub(a, v);
372}
373
374template<typename T>
375static T NoTsanAtomicFetchAnd(volatile T *a, T v, morder mo) {
376  return func_and(a, v);
377}
378
379template<typename T>
380static T NoTsanAtomicFetchOr(volatile T *a, T v, morder mo) {
381  return func_or(a, v);
382}
383
384template<typename T>
385static T NoTsanAtomicFetchXor(volatile T *a, T v, morder mo) {
386  return func_xor(a, v);
387}
388
389template<typename T>
390static T NoTsanAtomicFetchNand(volatile T *a, T v, morder mo) {
391  return func_nand(a, v);
392}
393
394template<typename T>
395static T AtomicExchange(ThreadState *thr, uptr pc, volatile T *a, T v,
396    morder mo) {
397  return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo);
398}
399
400template<typename T>
401static T AtomicFetchAdd(ThreadState *thr, uptr pc, volatile T *a, T v,
402    morder mo) {
403  return AtomicRMW<T, func_add>(thr, pc, a, v, mo);
404}
405
406template<typename T>
407static T AtomicFetchSub(ThreadState *thr, uptr pc, volatile T *a, T v,
408    morder mo) {
409  return AtomicRMW<T, func_sub>(thr, pc, a, v, mo);
410}
411
412template<typename T>
413static T AtomicFetchAnd(ThreadState *thr, uptr pc, volatile T *a, T v,
414    morder mo) {
415  return AtomicRMW<T, func_and>(thr, pc, a, v, mo);
416}
417
418template<typename T>
419static T AtomicFetchOr(ThreadState *thr, uptr pc, volatile T *a, T v,
420    morder mo) {
421  return AtomicRMW<T, func_or>(thr, pc, a, v, mo);
422}
423
424template<typename T>
425static T AtomicFetchXor(ThreadState *thr, uptr pc, volatile T *a, T v,
426    morder mo) {
427  return AtomicRMW<T, func_xor>(thr, pc, a, v, mo);
428}
429
430template<typename T>
431static T AtomicFetchNand(ThreadState *thr, uptr pc, volatile T *a, T v,
432    morder mo) {
433  return AtomicRMW<T, func_nand>(thr, pc, a, v, mo);
434}
435
436template<typename T>
437static bool NoTsanAtomicCAS(volatile T *a, T *c, T v, morder mo, morder fmo) {
438  return atomic_compare_exchange_strong(to_atomic(a), c, v, to_mo(mo));
439}
440
441#if __TSAN_HAS_INT128
442static bool NoTsanAtomicCAS(volatile a128 *a, a128 *c, a128 v,
443    morder mo, morder fmo) {
444  a128 old = *c;
445  a128 cur = func_cas(a, old, v);
446  if (cur == old)
447    return true;
448  *c = cur;
449  return false;
450}
451#endif
452
453template<typename T>
454static bool NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) {
455  return NoTsanAtomicCAS(a, &c, v, mo, fmo);
456}
457
458template<typename T>
459static bool AtomicCAS(ThreadState *thr, uptr pc,
460    volatile T *a, T *c, T v, morder mo, morder fmo) {
461  (void)fmo;  // Unused because llvm does not pass it yet.
462  MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
463  SyncVar *s = 0;
464  bool write_lock = mo != mo_acquire && mo != mo_consume;
465  if (mo != mo_relaxed) {
466    s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, write_lock);
467    thr->fast_state.IncrementEpoch();
468    // Can't increment epoch w/o writing to the trace as well.
469    TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
470    if (IsAcqRelOrder(mo))
471      AcquireReleaseImpl(thr, pc, &s->clock);
472    else if (IsReleaseOrder(mo))
473      ReleaseImpl(thr, pc, &s->clock);
474    else if (IsAcquireOrder(mo))
475      AcquireImpl(thr, pc, &s->clock);
476  }
477  T cc = *c;
478  T pr = func_cas(a, cc, v);
479  if (s) {
480    if (write_lock)
481      s->mtx.Unlock();
482    else
483      s->mtx.ReadUnlock();
484  }
485  if (pr == cc)
486    return true;
487  *c = pr;
488  return false;
489}
490
491template<typename T>
492static T AtomicCAS(ThreadState *thr, uptr pc,
493    volatile T *a, T c, T v, morder mo, morder fmo) {
494  AtomicCAS(thr, pc, a, &c, v, mo, fmo);
495  return c;
496}
497
498static void NoTsanAtomicFence(morder mo) {
499  __sync_synchronize();
500}
501
502static void AtomicFence(ThreadState *thr, uptr pc, morder mo) {
503  // FIXME(dvyukov): not implemented.
504  __sync_synchronize();
505}
506
507extern "C" {
508SANITIZER_INTERFACE_ATTRIBUTE
509a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) {
510  SCOPED_ATOMIC(Load, a, mo);
511}
512
513SANITIZER_INTERFACE_ATTRIBUTE
514a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) {
515  SCOPED_ATOMIC(Load, a, mo);
516}
517
518SANITIZER_INTERFACE_ATTRIBUTE
519a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) {
520  SCOPED_ATOMIC(Load, a, mo);
521}
522
523SANITIZER_INTERFACE_ATTRIBUTE
524a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) {
525  SCOPED_ATOMIC(Load, a, mo);
526}
527
528#if __TSAN_HAS_INT128
529SANITIZER_INTERFACE_ATTRIBUTE
530a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) {
531  SCOPED_ATOMIC(Load, a, mo);
532}
533#endif
534
535SANITIZER_INTERFACE_ATTRIBUTE
536void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) {
537  SCOPED_ATOMIC(Store, a, v, mo);
538}
539
540SANITIZER_INTERFACE_ATTRIBUTE
541void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) {
542  SCOPED_ATOMIC(Store, a, v, mo);
543}
544
545SANITIZER_INTERFACE_ATTRIBUTE
546void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) {
547  SCOPED_ATOMIC(Store, a, v, mo);
548}
549
550SANITIZER_INTERFACE_ATTRIBUTE
551void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) {
552  SCOPED_ATOMIC(Store, a, v, mo);
553}
554
555#if __TSAN_HAS_INT128
556SANITIZER_INTERFACE_ATTRIBUTE
557void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) {
558  SCOPED_ATOMIC(Store, a, v, mo);
559}
560#endif
561
562SANITIZER_INTERFACE_ATTRIBUTE
563a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) {
564  SCOPED_ATOMIC(Exchange, a, v, mo);
565}
566
567SANITIZER_INTERFACE_ATTRIBUTE
568a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) {
569  SCOPED_ATOMIC(Exchange, a, v, mo);
570}
571
572SANITIZER_INTERFACE_ATTRIBUTE
573a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) {
574  SCOPED_ATOMIC(Exchange, a, v, mo);
575}
576
577SANITIZER_INTERFACE_ATTRIBUTE
578a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) {
579  SCOPED_ATOMIC(Exchange, a, v, mo);
580}
581
582#if __TSAN_HAS_INT128
583SANITIZER_INTERFACE_ATTRIBUTE
584a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) {
585  SCOPED_ATOMIC(Exchange, a, v, mo);
586}
587#endif
588
589SANITIZER_INTERFACE_ATTRIBUTE
590a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) {
591  SCOPED_ATOMIC(FetchAdd, a, v, mo);
592}
593
594SANITIZER_INTERFACE_ATTRIBUTE
595a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) {
596  SCOPED_ATOMIC(FetchAdd, a, v, mo);
597}
598
599SANITIZER_INTERFACE_ATTRIBUTE
600a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) {
601  SCOPED_ATOMIC(FetchAdd, a, v, mo);
602}
603
604SANITIZER_INTERFACE_ATTRIBUTE
605a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) {
606  SCOPED_ATOMIC(FetchAdd, a, v, mo);
607}
608
609#if __TSAN_HAS_INT128
610SANITIZER_INTERFACE_ATTRIBUTE
611a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) {
612  SCOPED_ATOMIC(FetchAdd, a, v, mo);
613}
614#endif
615
616SANITIZER_INTERFACE_ATTRIBUTE
617a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) {
618  SCOPED_ATOMIC(FetchSub, a, v, mo);
619}
620
621SANITIZER_INTERFACE_ATTRIBUTE
622a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) {
623  SCOPED_ATOMIC(FetchSub, a, v, mo);
624}
625
626SANITIZER_INTERFACE_ATTRIBUTE
627a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) {
628  SCOPED_ATOMIC(FetchSub, a, v, mo);
629}
630
631SANITIZER_INTERFACE_ATTRIBUTE
632a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) {
633  SCOPED_ATOMIC(FetchSub, a, v, mo);
634}
635
636#if __TSAN_HAS_INT128
637SANITIZER_INTERFACE_ATTRIBUTE
638a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) {
639  SCOPED_ATOMIC(FetchSub, a, v, mo);
640}
641#endif
642
643SANITIZER_INTERFACE_ATTRIBUTE
644a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) {
645  SCOPED_ATOMIC(FetchAnd, a, v, mo);
646}
647
648SANITIZER_INTERFACE_ATTRIBUTE
649a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) {
650  SCOPED_ATOMIC(FetchAnd, a, v, mo);
651}
652
653SANITIZER_INTERFACE_ATTRIBUTE
654a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) {
655  SCOPED_ATOMIC(FetchAnd, a, v, mo);
656}
657
658SANITIZER_INTERFACE_ATTRIBUTE
659a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) {
660  SCOPED_ATOMIC(FetchAnd, a, v, mo);
661}
662
663#if __TSAN_HAS_INT128
664SANITIZER_INTERFACE_ATTRIBUTE
665a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) {
666  SCOPED_ATOMIC(FetchAnd, a, v, mo);
667}
668#endif
669
670SANITIZER_INTERFACE_ATTRIBUTE
671a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) {
672  SCOPED_ATOMIC(FetchOr, a, v, mo);
673}
674
675SANITIZER_INTERFACE_ATTRIBUTE
676a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) {
677  SCOPED_ATOMIC(FetchOr, a, v, mo);
678}
679
680SANITIZER_INTERFACE_ATTRIBUTE
681a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) {
682  SCOPED_ATOMIC(FetchOr, a, v, mo);
683}
684
685SANITIZER_INTERFACE_ATTRIBUTE
686a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) {
687  SCOPED_ATOMIC(FetchOr, a, v, mo);
688}
689
690#if __TSAN_HAS_INT128
691SANITIZER_INTERFACE_ATTRIBUTE
692a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) {
693  SCOPED_ATOMIC(FetchOr, a, v, mo);
694}
695#endif
696
697SANITIZER_INTERFACE_ATTRIBUTE
698a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) {
699  SCOPED_ATOMIC(FetchXor, a, v, mo);
700}
701
702SANITIZER_INTERFACE_ATTRIBUTE
703a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) {
704  SCOPED_ATOMIC(FetchXor, a, v, mo);
705}
706
707SANITIZER_INTERFACE_ATTRIBUTE
708a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) {
709  SCOPED_ATOMIC(FetchXor, a, v, mo);
710}
711
712SANITIZER_INTERFACE_ATTRIBUTE
713a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) {
714  SCOPED_ATOMIC(FetchXor, a, v, mo);
715}
716
717#if __TSAN_HAS_INT128
718SANITIZER_INTERFACE_ATTRIBUTE
719a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) {
720  SCOPED_ATOMIC(FetchXor, a, v, mo);
721}
722#endif
723
724SANITIZER_INTERFACE_ATTRIBUTE
725a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) {
726  SCOPED_ATOMIC(FetchNand, a, v, mo);
727}
728
729SANITIZER_INTERFACE_ATTRIBUTE
730a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) {
731  SCOPED_ATOMIC(FetchNand, a, v, mo);
732}
733
734SANITIZER_INTERFACE_ATTRIBUTE
735a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) {
736  SCOPED_ATOMIC(FetchNand, a, v, mo);
737}
738
739SANITIZER_INTERFACE_ATTRIBUTE
740a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) {
741  SCOPED_ATOMIC(FetchNand, a, v, mo);
742}
743
744#if __TSAN_HAS_INT128
745SANITIZER_INTERFACE_ATTRIBUTE
746a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) {
747  SCOPED_ATOMIC(FetchNand, a, v, mo);
748}
749#endif
750
751SANITIZER_INTERFACE_ATTRIBUTE
752int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v,
753    morder mo, morder fmo) {
754  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
755}
756
757SANITIZER_INTERFACE_ATTRIBUTE
758int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v,
759    morder mo, morder fmo) {
760  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
761}
762
763SANITIZER_INTERFACE_ATTRIBUTE
764int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v,
765    morder mo, morder fmo) {
766  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
767}
768
769SANITIZER_INTERFACE_ATTRIBUTE
770int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v,
771    morder mo, morder fmo) {
772  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
773}
774
775#if __TSAN_HAS_INT128
776SANITIZER_INTERFACE_ATTRIBUTE
777int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v,
778    morder mo, morder fmo) {
779  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
780}
781#endif
782
783SANITIZER_INTERFACE_ATTRIBUTE
784int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v,
785    morder mo, morder fmo) {
786  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
787}
788
789SANITIZER_INTERFACE_ATTRIBUTE
790int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v,
791    morder mo, morder fmo) {
792  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
793}
794
795SANITIZER_INTERFACE_ATTRIBUTE
796int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v,
797    morder mo, morder fmo) {
798  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
799}
800
801SANITIZER_INTERFACE_ATTRIBUTE
802int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v,
803    morder mo, morder fmo) {
804  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
805}
806
807#if __TSAN_HAS_INT128
808SANITIZER_INTERFACE_ATTRIBUTE
809int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v,
810    morder mo, morder fmo) {
811  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
812}
813#endif
814
815SANITIZER_INTERFACE_ATTRIBUTE
816a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v,
817    morder mo, morder fmo) {
818  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
819}
820
821SANITIZER_INTERFACE_ATTRIBUTE
822a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v,
823    morder mo, morder fmo) {
824  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
825}
826
827SANITIZER_INTERFACE_ATTRIBUTE
828a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v,
829    morder mo, morder fmo) {
830  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
831}
832
833SANITIZER_INTERFACE_ATTRIBUTE
834a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v,
835    morder mo, morder fmo) {
836  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
837}
838
839#if __TSAN_HAS_INT128
840SANITIZER_INTERFACE_ATTRIBUTE
841a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v,
842    morder mo, morder fmo) {
843  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
844}
845#endif
846
847SANITIZER_INTERFACE_ATTRIBUTE
848void __tsan_atomic_thread_fence(morder mo) {
849  char* a = 0;
850  SCOPED_ATOMIC(Fence, mo);
851}
852
853SANITIZER_INTERFACE_ATTRIBUTE
854void __tsan_atomic_signal_fence(morder mo) {
855}
856}  // extern "C"
857