1//===-- sanitizer_atomic_msvc.h ---------------------------------*- C++ -*-===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file is a part of ThreadSanitizer/AddressSanitizer runtime.
11// Not intended for direct inclusion. Include sanitizer_atomic.h.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef SANITIZER_ATOMIC_MSVC_H
16#define SANITIZER_ATOMIC_MSVC_H
17
18extern "C" void _ReadWriteBarrier();
19#pragma intrinsic(_ReadWriteBarrier)
20extern "C" void _mm_mfence();
21#pragma intrinsic(_mm_mfence)
22extern "C" void _mm_pause();
23#pragma intrinsic(_mm_pause)
24extern "C" char _InterlockedExchange8(   // NOLINT
25    char volatile *Addend, char Value);  // NOLINT
26#pragma intrinsic(_InterlockedExchange8)
27extern "C" short _InterlockedExchange16(   // NOLINT
28    short volatile *Addend, short Value);  // NOLINT
29#pragma intrinsic(_InterlockedExchange16)
30extern "C" long _InterlockedExchange(    // NOLINT
31    long volatile *Addend, long Value);  // NOLINT
32#pragma intrinsic(_InterlockedExchange)
33extern "C" long _InterlockedExchangeAdd(  // NOLINT
34    long volatile * Addend, long Value);  // NOLINT
35#pragma intrinsic(_InterlockedExchangeAdd)
36extern "C" char _InterlockedCompareExchange8(  // NOLINT
37    char volatile *Destination,                // NOLINT
38    char Exchange, char Comparand);            // NOLINT
39#pragma intrinsic(_InterlockedCompareExchange8)
40extern "C" short _InterlockedCompareExchange16(  // NOLINT
41    short volatile *Destination,                 // NOLINT
42    short Exchange, short Comparand);            // NOLINT
43#pragma intrinsic(_InterlockedCompareExchange16)
44extern "C"
45long long _InterlockedCompareExchange64(  // NOLINT
46    long long volatile *Destination,              // NOLINT
47    long long Exchange, long long Comparand);     // NOLINT
48#pragma intrinsic(_InterlockedCompareExchange64)
49extern "C" void *_InterlockedCompareExchangePointer(
50    void *volatile *Destination,
51    void *Exchange, void *Comparand);
52#pragma intrinsic(_InterlockedCompareExchangePointer)
53extern "C"
54long __cdecl _InterlockedCompareExchange(  // NOLINT
55    long volatile *Destination,            // NOLINT
56    long Exchange, long Comparand);        // NOLINT
57#pragma intrinsic(_InterlockedCompareExchange)
58
59#ifdef _WIN64
60extern "C" long long _InterlockedExchangeAdd64(     // NOLINT
61    long long volatile * Addend, long long Value);  // NOLINT
62#pragma intrinsic(_InterlockedExchangeAdd64)
63#endif
64
65namespace __sanitizer {
66
67INLINE void atomic_signal_fence(memory_order) {
68  _ReadWriteBarrier();
69}
70
71INLINE void atomic_thread_fence(memory_order) {
72  _mm_mfence();
73}
74
75INLINE void proc_yield(int cnt) {
76  for (int i = 0; i < cnt; i++)
77    _mm_pause();
78}
79
80template<typename T>
81INLINE typename T::Type atomic_load(
82    const volatile T *a, memory_order mo) {
83  DCHECK(mo & (memory_order_relaxed | memory_order_consume
84      | memory_order_acquire | memory_order_seq_cst));
85  DCHECK(!((uptr)a % sizeof(*a)));
86  typename T::Type v;
87  // FIXME(dvyukov): 64-bit load is not atomic on 32-bits.
88  if (mo == memory_order_relaxed) {
89    v = a->val_dont_use;
90  } else {
91    atomic_signal_fence(memory_order_seq_cst);
92    v = a->val_dont_use;
93    atomic_signal_fence(memory_order_seq_cst);
94  }
95  return v;
96}
97
98template<typename T>
99INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
100  DCHECK(mo & (memory_order_relaxed | memory_order_release
101      | memory_order_seq_cst));
102  DCHECK(!((uptr)a % sizeof(*a)));
103  // FIXME(dvyukov): 64-bit store is not atomic on 32-bits.
104  if (mo == memory_order_relaxed) {
105    a->val_dont_use = v;
106  } else {
107    atomic_signal_fence(memory_order_seq_cst);
108    a->val_dont_use = v;
109    atomic_signal_fence(memory_order_seq_cst);
110  }
111  if (mo == memory_order_seq_cst)
112    atomic_thread_fence(memory_order_seq_cst);
113}
114
115INLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a,
116    u32 v, memory_order mo) {
117  (void)mo;
118  DCHECK(!((uptr)a % sizeof(*a)));
119  return (u32)_InterlockedExchangeAdd(
120      (volatile long*)&a->val_dont_use, (long)v);  // NOLINT
121}
122
123INLINE uptr atomic_fetch_add(volatile atomic_uintptr_t *a,
124    uptr v, memory_order mo) {
125  (void)mo;
126  DCHECK(!((uptr)a % sizeof(*a)));
127#ifdef _WIN64
128  return (uptr)_InterlockedExchangeAdd64(
129      (volatile long long*)&a->val_dont_use, (long long)v);  // NOLINT
130#else
131  return (uptr)_InterlockedExchangeAdd(
132      (volatile long*)&a->val_dont_use, (long)v);  // NOLINT
133#endif
134}
135
136INLINE u32 atomic_fetch_sub(volatile atomic_uint32_t *a,
137    u32 v, memory_order mo) {
138  (void)mo;
139  DCHECK(!((uptr)a % sizeof(*a)));
140  return (u32)_InterlockedExchangeAdd(
141      (volatile long*)&a->val_dont_use, -(long)v);  // NOLINT
142}
143
144INLINE uptr atomic_fetch_sub(volatile atomic_uintptr_t *a,
145    uptr v, memory_order mo) {
146  (void)mo;
147  DCHECK(!((uptr)a % sizeof(*a)));
148#ifdef _WIN64
149  return (uptr)_InterlockedExchangeAdd64(
150      (volatile long long*)&a->val_dont_use, -(long long)v);  // NOLINT
151#else
152  return (uptr)_InterlockedExchangeAdd(
153      (volatile long*)&a->val_dont_use, -(long)v);  // NOLINT
154#endif
155}
156
157INLINE u8 atomic_exchange(volatile atomic_uint8_t *a,
158    u8 v, memory_order mo) {
159  (void)mo;
160  DCHECK(!((uptr)a % sizeof(*a)));
161  return (u8)_InterlockedExchange8((volatile char*)&a->val_dont_use, v);
162}
163
164INLINE u16 atomic_exchange(volatile atomic_uint16_t *a,
165    u16 v, memory_order mo) {
166  (void)mo;
167  DCHECK(!((uptr)a % sizeof(*a)));
168  return (u16)_InterlockedExchange16((volatile short*)&a->val_dont_use, v);
169}
170
171INLINE u32 atomic_exchange(volatile atomic_uint32_t *a,
172    u32 v, memory_order mo) {
173  (void)mo;
174  DCHECK(!((uptr)a % sizeof(*a)));
175  return (u32)_InterlockedExchange((volatile long*)&a->val_dont_use, v);
176}
177
178INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a,
179                                           u8 *cmp,
180                                           u8 xchgv,
181                                           memory_order mo) {
182  (void)mo;
183  DCHECK(!((uptr)a % sizeof(*a)));
184  u8 cmpv = *cmp;
185#ifdef _WIN64
186  u8 prev = (u8)_InterlockedCompareExchange8(
187      (volatile char*)&a->val_dont_use, (char)xchgv, (char)cmpv);
188#else
189  u8 prev;
190  __asm {
191    mov al, cmpv
192    mov ecx, a
193    mov dl, xchgv
194    lock cmpxchg [ecx], dl
195    mov prev, al
196  }
197#endif
198  if (prev == cmpv)
199    return true;
200  *cmp = prev;
201  return false;
202}
203
204INLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a,
205                                           uptr *cmp,
206                                           uptr xchg,
207                                           memory_order mo) {
208  uptr cmpv = *cmp;
209  uptr prev = (uptr)_InterlockedCompareExchangePointer(
210      (void*volatile*)&a->val_dont_use, (void*)xchg, (void*)cmpv);
211  if (prev == cmpv)
212    return true;
213  *cmp = prev;
214  return false;
215}
216
217INLINE bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a,
218                                           u16 *cmp,
219                                           u16 xchg,
220                                           memory_order mo) {
221  u16 cmpv = *cmp;
222  u16 prev = (u16)_InterlockedCompareExchange16(
223      (volatile short*)&a->val_dont_use, (short)xchg, (short)cmpv);
224  if (prev == cmpv)
225    return true;
226  *cmp = prev;
227  return false;
228}
229
230INLINE bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a,
231                                           u32 *cmp,
232                                           u32 xchg,
233                                           memory_order mo) {
234  u32 cmpv = *cmp;
235  u32 prev = (u32)_InterlockedCompareExchange(
236      (volatile long*)&a->val_dont_use, (long)xchg, (long)cmpv);
237  if (prev == cmpv)
238    return true;
239  *cmp = prev;
240  return false;
241}
242
243INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a,
244                                           u64 *cmp,
245                                           u64 xchg,
246                                           memory_order mo) {
247  u64 cmpv = *cmp;
248  u64 prev = (u64)_InterlockedCompareExchange64(
249      (volatile long long*)&a->val_dont_use, (long long)xchg, (long long)cmpv);
250  if (prev == cmpv)
251    return true;
252  *cmp = prev;
253  return false;
254}
255
256template<typename T>
257INLINE bool atomic_compare_exchange_weak(volatile T *a,
258                                         typename T::Type *cmp,
259                                         typename T::Type xchg,
260                                         memory_order mo) {
261  return atomic_compare_exchange_strong(a, cmp, xchg, mo);
262}
263
264}  // namespace __sanitizer
265
266#endif  // SANITIZER_ATOMIC_CLANG_H
267