1b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov//===-- sanitizer_atomic_msvc.h ---------------------------------*- C++ -*-===// 2b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov// 3b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov// The LLVM Compiler Infrastructure 4b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov// 5b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov// This file is distributed under the University of Illinois Open Source 6b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov// License. See LICENSE.TXT for details. 7b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov// 8b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov//===----------------------------------------------------------------------===// 9b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov// 10b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov// This file is a part of ThreadSanitizer/AddressSanitizer runtime. 11b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov// Not intended for direct inclusion. Include sanitizer_atomic.h. 12b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov// 13b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov//===----------------------------------------------------------------------===// 14b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov 15b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov#ifndef SANITIZER_ATOMIC_MSVC_H 16b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov#define SANITIZER_ATOMIC_MSVC_H 17b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov 1869021355c0ac8a0d9b546fb3db1136b1b2db0616Dmitry Vyukovextern "C" void _ReadWriteBarrier(); 196dab1903fb38dfdee287fb4a6e045143a19189bcDmitry Vyukov#pragma intrinsic(_ReadWriteBarrier) 2069021355c0ac8a0d9b546fb3db1136b1b2db0616Dmitry Vyukovextern "C" void _mm_mfence(); 216dab1903fb38dfdee287fb4a6e045143a19189bcDmitry Vyukov#pragma intrinsic(_mm_mfence) 2269021355c0ac8a0d9b546fb3db1136b1b2db0616Dmitry Vyukovextern "C" void _mm_pause(); 236dab1903fb38dfdee287fb4a6e045143a19189bcDmitry Vyukov#pragma intrinsic(_mm_pause) 2469021355c0ac8a0d9b546fb3db1136b1b2db0616Dmitry Vyukovextern "C" long _InterlockedExchangeAdd( // NOLINT 2569021355c0ac8a0d9b546fb3db1136b1b2db0616Dmitry Vyukov long volatile * Addend, long Value); // NOLINT 266dab1903fb38dfdee287fb4a6e045143a19189bcDmitry Vyukov#pragma intrinsic(_InterlockedExchangeAdd) 272d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesextern "C" short _InterlockedCompareExchange16( // NOLINT 282d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines short volatile *Destination, // NOLINT 292d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines short Exchange, short Comparand); // NOLINT 302d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#pragma intrinsic(_InterlockedCompareExchange16) 312d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesextern "C" 322d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hineslong long _InterlockedCompareExchange64( // NOLINT 332d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines long long volatile *Destination, // NOLINT 342d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines long long Exchange, long long Comparand); // NOLINT 352d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#pragma intrinsic(_InterlockedCompareExchange64) 36806c491ee17ed34c6385e84e416be15ed9521f3cAlexey Samsonovextern "C" void *_InterlockedCompareExchangePointer( 370d9c08d74cb5e697c5e3ad7ddfb9f5a414d249dfDmitry Vyukov void *volatile *Destination, 380d9c08d74cb5e697c5e3ad7ddfb9f5a414d249dfDmitry Vyukov void *Exchange, void *Comparand); 39806c491ee17ed34c6385e84e416be15ed9521f3cAlexey Samsonov#pragma intrinsic(_InterlockedCompareExchangePointer) 400cf7eefe0e7cafd357961e91e2efb7f7f363bf19Timur Iskhodzhanovextern "C" 41b1d1ef27db5b6c7788cda3dfd5deb6037dd4a3b8Timur Iskhodzhanovlong __cdecl _InterlockedCompareExchange( // NOLINT 42b1d1ef27db5b6c7788cda3dfd5deb6037dd4a3b8Timur Iskhodzhanov long volatile *Destination, // NOLINT 43b1d1ef27db5b6c7788cda3dfd5deb6037dd4a3b8Timur Iskhodzhanov long Exchange, long Comparand); // NOLINT 440cf7eefe0e7cafd357961e91e2efb7f7f363bf19Timur Iskhodzhanov#pragma intrinsic(_InterlockedCompareExchange) 450cf7eefe0e7cafd357961e91e2efb7f7f363bf19Timur Iskhodzhanov 465d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines#ifdef _WIN64 475d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hinesextern "C" long long _InterlockedExchangeAdd64( // NOLINT 485d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines long long volatile * Addend, long long Value); // NOLINT 495d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines#pragma intrinsic(_InterlockedExchangeAdd64) 500cf7eefe0e7cafd357961e91e2efb7f7f363bf19Timur Iskhodzhanov#endif 51b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov 52b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukovnamespace __sanitizer { 53b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov 54b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry VyukovINLINE void atomic_signal_fence(memory_order) { 55b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov _ReadWriteBarrier(); 56b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov} 57b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov 58b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry VyukovINLINE void atomic_thread_fence(memory_order) { 59b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov _mm_mfence(); 60b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov} 61b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov 62b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry VyukovINLINE void proc_yield(int cnt) { 63b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov for (int i = 0; i < cnt; i++) 64b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov _mm_pause(); 65b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov} 66b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov 67b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukovtemplate<typename T> 68b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry VyukovINLINE typename T::Type atomic_load( 69b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov const volatile T *a, memory_order mo) { 70b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov DCHECK(mo & (memory_order_relaxed | memory_order_consume 71b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov | memory_order_acquire | memory_order_seq_cst)); 72b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov DCHECK(!((uptr)a % sizeof(*a))); 73b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov typename T::Type v; 741a3503b1c6c707705bf2ebc33343da0b04e3a26dDmitry Vyukov // FIXME(dvyukov): 64-bit load is not atomic on 32-bits. 75b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov if (mo == memory_order_relaxed) { 76b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov v = a->val_dont_use; 77b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov } else { 78b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov atomic_signal_fence(memory_order_seq_cst); 79b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov v = a->val_dont_use; 80b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov atomic_signal_fence(memory_order_seq_cst); 81b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov } 82b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov return v; 83b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov} 84b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov 85b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukovtemplate<typename T> 86b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry VyukovINLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) { 87b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov DCHECK(mo & (memory_order_relaxed | memory_order_release 88b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov | memory_order_seq_cst)); 89b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov DCHECK(!((uptr)a % sizeof(*a))); 901a3503b1c6c707705bf2ebc33343da0b04e3a26dDmitry Vyukov // FIXME(dvyukov): 64-bit store is not atomic on 32-bits. 91b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov if (mo == memory_order_relaxed) { 92b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov a->val_dont_use = v; 93b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov } else { 94b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov atomic_signal_fence(memory_order_seq_cst); 95b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov a->val_dont_use = v; 96b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov atomic_signal_fence(memory_order_seq_cst); 97b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov } 98b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov if (mo == memory_order_seq_cst) 99b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov atomic_thread_fence(memory_order_seq_cst); 100b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov} 101b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov 102b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry VyukovINLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a, 103b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov u32 v, memory_order mo) { 104b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov (void)mo; 105b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov DCHECK(!((uptr)a % sizeof(*a))); 106b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov return (u32)_InterlockedExchangeAdd( 107b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov (volatile long*)&a->val_dont_use, (long)v); // NOLINT 108b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov} 109b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov 1102d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesINLINE uptr atomic_fetch_add(volatile atomic_uintptr_t *a, 1112d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines uptr v, memory_order mo) { 1122d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines (void)mo; 1132d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines DCHECK(!((uptr)a % sizeof(*a))); 1142d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#ifdef _WIN64 1152d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return (uptr)_InterlockedExchangeAdd64( 1162d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines (volatile long long*)&a->val_dont_use, (long long)v); // NOLINT 1172d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#else 1182d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return (uptr)_InterlockedExchangeAdd( 1192d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines (volatile long*)&a->val_dont_use, (long)v); // NOLINT 1202d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#endif 1212d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 1222d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 1232d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesINLINE u32 atomic_fetch_sub(volatile atomic_uint32_t *a, 1242d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines u32 v, memory_order mo) { 1252d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines (void)mo; 1262d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines DCHECK(!((uptr)a % sizeof(*a))); 1272d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return (u32)_InterlockedExchangeAdd( 1282d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines (volatile long*)&a->val_dont_use, -(long)v); // NOLINT 1292d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 1302d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 1312d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesINLINE uptr atomic_fetch_sub(volatile atomic_uintptr_t *a, 1322d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines uptr v, memory_order mo) { 1332d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines (void)mo; 1342d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines DCHECK(!((uptr)a % sizeof(*a))); 1352d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#ifdef _WIN64 1362d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return (uptr)_InterlockedExchangeAdd64( 1372d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines (volatile long long*)&a->val_dont_use, -(long long)v); // NOLINT 1382d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#else 1392d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return (uptr)_InterlockedExchangeAdd( 1402d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines (volatile long*)&a->val_dont_use, -(long)v); // NOLINT 1412d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#endif 1422d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 1432d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 144b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry VyukovINLINE u8 atomic_exchange(volatile atomic_uint8_t *a, 145b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov u8 v, memory_order mo) { 146b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov (void)mo; 147b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov DCHECK(!((uptr)a % sizeof(*a))); 148b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov __asm { 149b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov mov eax, a 1506dab1903fb38dfdee287fb4a6e045143a19189bcDmitry Vyukov mov cl, v 1516dab1903fb38dfdee287fb4a6e045143a19189bcDmitry Vyukov xchg [eax], cl // NOLINT 1526dab1903fb38dfdee287fb4a6e045143a19189bcDmitry Vyukov mov v, cl 153b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov } 154b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov return v; 155b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov} 156b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov 157b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry VyukovINLINE u16 atomic_exchange(volatile atomic_uint16_t *a, 158b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov u16 v, memory_order mo) { 159b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov (void)mo; 160b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov DCHECK(!((uptr)a % sizeof(*a))); 161b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov __asm { 162b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov mov eax, a 1636dab1903fb38dfdee287fb4a6e045143a19189bcDmitry Vyukov mov cx, v 1646dab1903fb38dfdee287fb4a6e045143a19189bcDmitry Vyukov xchg [eax], cx // NOLINT 1656dab1903fb38dfdee287fb4a6e045143a19189bcDmitry Vyukov mov v, cx 166b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov } 167b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov return v; 168b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov} 169b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov 170b975c8abce50e513d9e168b24692fa9310e32aedAlexey SamsonovINLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a, 171b975c8abce50e513d9e168b24692fa9310e32aedAlexey Samsonov u8 *cmp, 17240b631f03580b3c631f332c1f4ebfbed453b4283Timur Iskhodzhanov u8 xchgv, 173b975c8abce50e513d9e168b24692fa9310e32aedAlexey Samsonov memory_order mo) { 174b975c8abce50e513d9e168b24692fa9310e32aedAlexey Samsonov (void)mo; 175b975c8abce50e513d9e168b24692fa9310e32aedAlexey Samsonov DCHECK(!((uptr)a % sizeof(*a))); 176b975c8abce50e513d9e168b24692fa9310e32aedAlexey Samsonov u8 cmpv = *cmp; 177b975c8abce50e513d9e168b24692fa9310e32aedAlexey Samsonov u8 prev; 178b975c8abce50e513d9e168b24692fa9310e32aedAlexey Samsonov __asm { 179b975c8abce50e513d9e168b24692fa9310e32aedAlexey Samsonov mov al, cmpv 180b975c8abce50e513d9e168b24692fa9310e32aedAlexey Samsonov mov ecx, a 18140b631f03580b3c631f332c1f4ebfbed453b4283Timur Iskhodzhanov mov dl, xchgv 182b975c8abce50e513d9e168b24692fa9310e32aedAlexey Samsonov lock cmpxchg [ecx], dl 183b975c8abce50e513d9e168b24692fa9310e32aedAlexey Samsonov mov prev, al 184b975c8abce50e513d9e168b24692fa9310e32aedAlexey Samsonov } 185b975c8abce50e513d9e168b24692fa9310e32aedAlexey Samsonov if (prev == cmpv) 186b975c8abce50e513d9e168b24692fa9310e32aedAlexey Samsonov return true; 187b975c8abce50e513d9e168b24692fa9310e32aedAlexey Samsonov *cmp = prev; 188b975c8abce50e513d9e168b24692fa9310e32aedAlexey Samsonov return false; 189b975c8abce50e513d9e168b24692fa9310e32aedAlexey Samsonov} 190b975c8abce50e513d9e168b24692fa9310e32aedAlexey Samsonov 1911b1de03d493b5d4950988924ee4ad900b6c78008Dmitry VyukovINLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a, 1920d9c08d74cb5e697c5e3ad7ddfb9f5a414d249dfDmitry Vyukov uptr *cmp, 1930d9c08d74cb5e697c5e3ad7ddfb9f5a414d249dfDmitry Vyukov uptr xchg, 1940d9c08d74cb5e697c5e3ad7ddfb9f5a414d249dfDmitry Vyukov memory_order mo) { 1950d9c08d74cb5e697c5e3ad7ddfb9f5a414d249dfDmitry Vyukov uptr cmpv = *cmp; 196806c491ee17ed34c6385e84e416be15ed9521f3cAlexey Samsonov uptr prev = (uptr)_InterlockedCompareExchangePointer( 1970d9c08d74cb5e697c5e3ad7ddfb9f5a414d249dfDmitry Vyukov (void*volatile*)&a->val_dont_use, (void*)xchg, (void*)cmpv); 1980d9c08d74cb5e697c5e3ad7ddfb9f5a414d249dfDmitry Vyukov if (prev == cmpv) 1990d9c08d74cb5e697c5e3ad7ddfb9f5a414d249dfDmitry Vyukov return true; 2000d9c08d74cb5e697c5e3ad7ddfb9f5a414d249dfDmitry Vyukov *cmp = prev; 2010d9c08d74cb5e697c5e3ad7ddfb9f5a414d249dfDmitry Vyukov return false; 2020d9c08d74cb5e697c5e3ad7ddfb9f5a414d249dfDmitry Vyukov} 2030d9c08d74cb5e697c5e3ad7ddfb9f5a414d249dfDmitry Vyukov 2042d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesINLINE bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a, 2052d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines u16 *cmp, 2062d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines u16 xchg, 2072d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines memory_order mo) { 2082d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines u16 cmpv = *cmp; 2092d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines u16 prev = (u16)_InterlockedCompareExchange16( 2102d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines (volatile short*)&a->val_dont_use, (short)xchg, (short)cmpv); 2112d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (prev == cmpv) 2122d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return true; 2132d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines *cmp = prev; 2142d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return false; 2152d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 2162d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 2172d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesINLINE bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a, 2182d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines u32 *cmp, 2192d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines u32 xchg, 2202d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines memory_order mo) { 2212d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines u32 cmpv = *cmp; 2222d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines u32 prev = (u32)_InterlockedCompareExchange( 2232d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines (volatile long*)&a->val_dont_use, (long)xchg, (long)cmpv); 2242d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (prev == cmpv) 2252d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return true; 2262d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines *cmp = prev; 2272d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return false; 2282d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 2292d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 2302d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesINLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a, 2312d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines u64 *cmp, 2322d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines u64 xchg, 2332d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines memory_order mo) { 2342d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines u64 cmpv = *cmp; 2352d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines u64 prev = (u64)_InterlockedCompareExchange64( 2362d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines (volatile long long*)&a->val_dont_use, (long long)xchg, (long long)cmpv); 2372d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (prev == cmpv) 2382d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return true; 2392d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines *cmp = prev; 2402d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return false; 2412d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 2422d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 2430d9c08d74cb5e697c5e3ad7ddfb9f5a414d249dfDmitry Vyukovtemplate<typename T> 2440d9c08d74cb5e697c5e3ad7ddfb9f5a414d249dfDmitry VyukovINLINE bool atomic_compare_exchange_weak(volatile T *a, 245b975c8abce50e513d9e168b24692fa9310e32aedAlexey Samsonov typename T::Type *cmp, 246b975c8abce50e513d9e168b24692fa9310e32aedAlexey Samsonov typename T::Type xchg, 247b975c8abce50e513d9e168b24692fa9310e32aedAlexey Samsonov memory_order mo) { 2480d9c08d74cb5e697c5e3ad7ddfb9f5a414d249dfDmitry Vyukov return atomic_compare_exchange_strong(a, cmp, xchg, mo); 2490d9c08d74cb5e697c5e3ad7ddfb9f5a414d249dfDmitry Vyukov} 2500d9c08d74cb5e697c5e3ad7ddfb9f5a414d249dfDmitry Vyukov 251b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov} // namespace __sanitizer 252b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov 253b6eb56f5a03b4dba8de79465274a73d1ecbca7f2Dmitry Vyukov#endif // SANITIZER_ATOMIC_CLANG_H 254