sanitizer_atomic_msvc.h revision 5d71de26cedae3dafc17449fe0182045c0bd20e8 
1//===-- sanitizer_atomic_msvc.h ---------------------------------*- C++ -*-===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file is a part of ThreadSanitizer/AddressSanitizer runtime.
11// Not intended for direct inclusion. Include sanitizer_atomic.h.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef SANITIZER_ATOMIC_MSVC_H
16#define SANITIZER_ATOMIC_MSVC_H
17
18extern "C" void _ReadWriteBarrier();
19#pragma intrinsic(_ReadWriteBarrier)
20extern "C" void _mm_mfence();
21#pragma intrinsic(_mm_mfence)
22extern "C" void _mm_pause();
23#pragma intrinsic(_mm_pause)
24extern "C" long _InterlockedExchangeAdd(  // NOLINT
25    long volatile * Addend, long Value);  // NOLINT
26#pragma intrinsic(_InterlockedExchangeAdd)
27extern "C" short _InterlockedCompareExchange16(  // NOLINT
28    short volatile *Destination,                 // NOLINT
29    short Exchange, short Comparand);            // NOLINT
30#pragma intrinsic(_InterlockedCompareExchange16)
31extern "C"
32long long _InterlockedCompareExchange64(  // NOLINT
33    long long volatile *Destination,              // NOLINT
34    long long Exchange, long long Comparand);     // NOLINT
35#pragma intrinsic(_InterlockedCompareExchange64)
36extern "C" void *_InterlockedCompareExchangePointer(
37    void *volatile *Destination,
38    void *Exchange, void *Comparand);
39#pragma intrinsic(_InterlockedCompareExchangePointer)
40extern "C"
41long __cdecl _InterlockedCompareExchange(  // NOLINT
42    long volatile *Destination,            // NOLINT
43    long Exchange, long Comparand);        // NOLINT
44#pragma intrinsic(_InterlockedCompareExchange)
45
46#ifdef _WIN64
47extern "C" long long _InterlockedExchangeAdd64(     // NOLINT
48    long long volatile * Addend, long long Value);  // NOLINT
49#pragma intrinsic(_InterlockedExchangeAdd64)
50#endif
51
52namespace __sanitizer {
53
54INLINE void atomic_signal_fence(memory_order) {
55  _ReadWriteBarrier();
56}
57
58INLINE void atomic_thread_fence(memory_order) {
59  _mm_mfence();
60}
61
62INLINE void proc_yield(int cnt) {
63  for (int i = 0; i < cnt; i++)
64    _mm_pause();
65}
66
67template<typename T>
68INLINE typename T::Type atomic_load(
69    const volatile T *a, memory_order mo) {
70  DCHECK(mo & (memory_order_relaxed | memory_order_consume
71      | memory_order_acquire | memory_order_seq_cst));
72  DCHECK(!((uptr)a % sizeof(*a)));
73  typename T::Type v;
74  // FIXME(dvyukov): 64-bit load is not atomic on 32-bits.
75  if (mo == memory_order_relaxed) {
76    v = a->val_dont_use;
77  } else {
78    atomic_signal_fence(memory_order_seq_cst);
79    v = a->val_dont_use;
80    atomic_signal_fence(memory_order_seq_cst);
81  }
82  return v;
83}
84
85template<typename T>
86INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
87  DCHECK(mo & (memory_order_relaxed | memory_order_release
88      | memory_order_seq_cst));
89  DCHECK(!((uptr)a % sizeof(*a)));
90  // FIXME(dvyukov): 64-bit store is not atomic on 32-bits.
91  if (mo == memory_order_relaxed) {
92    a->val_dont_use = v;
93  } else {
94    atomic_signal_fence(memory_order_seq_cst);
95    a->val_dont_use = v;
96    atomic_signal_fence(memory_order_seq_cst);
97  }
98  if (mo == memory_order_seq_cst)
99    atomic_thread_fence(memory_order_seq_cst);
100}
101
102INLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a,
103    u32 v, memory_order mo) {
104  (void)mo;
105  DCHECK(!((uptr)a % sizeof(*a)));
106  return (u32)_InterlockedExchangeAdd(
107      (volatile long*)&a->val_dont_use, (long)v);  // NOLINT
108}
109
110INLINE uptr atomic_fetch_add(volatile atomic_uintptr_t *a,
111    uptr v, memory_order mo) {
112  (void)mo;
113  DCHECK(!((uptr)a % sizeof(*a)));
114#ifdef _WIN64
115  return (uptr)_InterlockedExchangeAdd64(
116      (volatile long long*)&a->val_dont_use, (long long)v);  // NOLINT
117#else
118  return (uptr)_InterlockedExchangeAdd(
119      (volatile long*)&a->val_dont_use, (long)v);  // NOLINT
120#endif
121}
122
123INLINE u32 atomic_fetch_sub(volatile atomic_uint32_t *a,
124    u32 v, memory_order mo) {
125  (void)mo;
126  DCHECK(!((uptr)a % sizeof(*a)));
127  return (u32)_InterlockedExchangeAdd(
128      (volatile long*)&a->val_dont_use, -(long)v);  // NOLINT
129}
130
131INLINE uptr atomic_fetch_sub(volatile atomic_uintptr_t *a,
132    uptr v, memory_order mo) {
133  (void)mo;
134  DCHECK(!((uptr)a % sizeof(*a)));
135#ifdef _WIN64
136  return (uptr)_InterlockedExchangeAdd64(
137      (volatile long long*)&a->val_dont_use, -(long long)v);  // NOLINT
138#else
139  return (uptr)_InterlockedExchangeAdd(
140      (volatile long*)&a->val_dont_use, -(long)v);  // NOLINT
141#endif
142}
143
144INLINE u8 atomic_exchange(volatile atomic_uint8_t *a,
145    u8 v, memory_order mo) {
146  (void)mo;
147  DCHECK(!((uptr)a % sizeof(*a)));
148  __asm {
149    mov eax, a
150    mov cl, v
151    xchg [eax], cl  // NOLINT
152    mov v, cl
153  }
154  return v;
155}
156
157INLINE u16 atomic_exchange(volatile atomic_uint16_t *a,
158    u16 v, memory_order mo) {
159  (void)mo;
160  DCHECK(!((uptr)a % sizeof(*a)));
161  __asm {
162    mov eax, a
163    mov cx, v
164    xchg [eax], cx  // NOLINT
165    mov v, cx
166  }
167  return v;
168}
169
170INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a,
171                                           u8 *cmp,
172                                           u8 xchgv,
173                                           memory_order mo) {
174  (void)mo;
175  DCHECK(!((uptr)a % sizeof(*a)));
176  u8 cmpv = *cmp;
177  u8 prev;
178  __asm {
179    mov al, cmpv
180    mov ecx, a
181    mov dl, xchgv
182    lock cmpxchg [ecx], dl
183    mov prev, al
184  }
185  if (prev == cmpv)
186    return true;
187  *cmp = prev;
188  return false;
189}
190
191INLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a,
192                                           uptr *cmp,
193                                           uptr xchg,
194                                           memory_order mo) {
195  uptr cmpv = *cmp;
196  uptr prev = (uptr)_InterlockedCompareExchangePointer(
197      (void*volatile*)&a->val_dont_use, (void*)xchg, (void*)cmpv);
198  if (prev == cmpv)
199    return true;
200  *cmp = prev;
201  return false;
202}
203
204INLINE bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a,
205                                           u16 *cmp,
206                                           u16 xchg,
207                                           memory_order mo) {
208  u16 cmpv = *cmp;
209  u16 prev = (u16)_InterlockedCompareExchange16(
210      (volatile short*)&a->val_dont_use, (short)xchg, (short)cmpv);
211  if (prev == cmpv)
212    return true;
213  *cmp = prev;
214  return false;
215}
216
217INLINE bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a,
218                                           u32 *cmp,
219                                           u32 xchg,
220                                           memory_order mo) {
221  u32 cmpv = *cmp;
222  u32 prev = (u32)_InterlockedCompareExchange(
223      (volatile long*)&a->val_dont_use, (long)xchg, (long)cmpv);
224  if (prev == cmpv)
225    return true;
226  *cmp = prev;
227  return false;
228}
229
230INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a,
231                                           u64 *cmp,
232                                           u64 xchg,
233                                           memory_order mo) {
234  u64 cmpv = *cmp;
235  u64 prev = (u64)_InterlockedCompareExchange64(
236      (volatile long long*)&a->val_dont_use, (long long)xchg, (long long)cmpv);
237  if (prev == cmpv)
238    return true;
239  *cmp = prev;
240  return false;
241}
242
243template<typename T>
244INLINE bool atomic_compare_exchange_weak(volatile T *a,
245                                         typename T::Type *cmp,
246                                         typename T::Type xchg,
247                                         memory_order mo) {
248  return atomic_compare_exchange_strong(a, cmp, xchg, mo);
249}
250
251}  // namespace __sanitizer
252
253#endif  // SANITIZER_ATOMIC_CLANG_H
254