19682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall//===-- sanitizer_atomic_msvc.h ---------------------------------*- C++ -*-===// 29682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall// 39682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall// The LLVM Compiler Infrastructure 49682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall// 59682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall// This file is distributed under the University of Illinois Open Source 69682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall// License. See LICENSE.TXT for details. 79682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall// 89682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall//===----------------------------------------------------------------------===// 99682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall// 109682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall// This file is a part of ThreadSanitizer/AddressSanitizer runtime. 119682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall// Not intended for direct inclusion. Include sanitizer_atomic.h. 129682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall// 139682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall//===----------------------------------------------------------------------===// 149682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall 159682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall#ifndef SANITIZER_ATOMIC_MSVC_H 169682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall#define SANITIZER_ATOMIC_MSVC_H 179682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall 189682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hallextern "C" void _ReadWriteBarrier(); 199682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall#pragma intrinsic(_ReadWriteBarrier) 209682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hallextern "C" void _mm_mfence(); 219682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall#pragma intrinsic(_mm_mfence) 229682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hallextern "C" void _mm_pause(); 239682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall#pragma intrinsic(_mm_pause) 249682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hallextern "C" long _InterlockedExchangeAdd( // NOLINT 259682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall long volatile * Addend, long Value); // NOLINT 269682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall#pragma intrinsic(_InterlockedExchangeAdd) 279682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hallextern "C" short _InterlockedCompareExchange16( // NOLINT 289682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall short volatile *Destination, // NOLINT 299682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall short Exchange, short Comparand); // NOLINT 309682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall#pragma intrinsic(_InterlockedCompareExchange16) 319682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hallextern "C" 329682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Halllong long _InterlockedCompareExchange64( // NOLINT 339682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall long long volatile *Destination, // NOLINT 349682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall long long Exchange, long long Comparand); // NOLINT 359682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall#pragma intrinsic(_InterlockedCompareExchange64) 369682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hallextern "C" void *_InterlockedCompareExchangePointer( 379682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall void *volatile *Destination, 389682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall void *Exchange, void *Comparand); 399682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall#pragma intrinsic(_InterlockedCompareExchangePointer) 409682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hallextern "C" 419682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Halllong __cdecl _InterlockedCompareExchange( // NOLINT 429682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall long volatile *Destination, // NOLINT 439682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall long Exchange, long Comparand); // NOLINT 449682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall#pragma intrinsic(_InterlockedCompareExchange) 459682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall 469682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall#ifdef _WIN64 479682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hallextern "C" long long _InterlockedExchangeAdd64( // NOLINT 489682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall long long volatile * Addend, long long Value); // NOLINT 499682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall#pragma intrinsic(_InterlockedExchangeAdd64) 509682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall#endif 519682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall 529682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hallnamespace __sanitizer { 539682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall 549682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse HallINLINE void atomic_signal_fence(memory_order) { 559682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall _ReadWriteBarrier(); 569682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall} 579682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall 589682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse HallINLINE void atomic_thread_fence(memory_order) { 599682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall _mm_mfence(); 609682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall} 619682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall 629682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse HallINLINE void proc_yield(int cnt) { 639682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall for (int i = 0; i < cnt; i++) 649682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall _mm_pause(); 659682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall} 669682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall 679682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Halltemplate<typename T> 689682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse HallINLINE typename T::Type atomic_load( 699682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall const volatile T *a, memory_order mo) { 709682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall DCHECK(mo & (memory_order_relaxed | memory_order_consume 719682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall | memory_order_acquire | memory_order_seq_cst)); 729682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall DCHECK(!((uptr)a % sizeof(*a))); 739682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall typename T::Type v; 749682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall // FIXME(dvyukov): 64-bit load is not atomic on 32-bits. 759682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall if (mo == memory_order_relaxed) { 769682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall v = a->val_dont_use; 779682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall } else { 789682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall atomic_signal_fence(memory_order_seq_cst); 799682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall v = a->val_dont_use; 809682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall atomic_signal_fence(memory_order_seq_cst); 819682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall } 829682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall return v; 839682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall} 849682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall 859682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Halltemplate<typename T> 869682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse HallINLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) { 879682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall DCHECK(mo & (memory_order_relaxed | memory_order_release 889682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall | memory_order_seq_cst)); 899682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall DCHECK(!((uptr)a % sizeof(*a))); 909682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall // FIXME(dvyukov): 64-bit store is not atomic on 32-bits. 919682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall if (mo == memory_order_relaxed) { 929682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall a->val_dont_use = v; 939682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall } else { 949682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall atomic_signal_fence(memory_order_seq_cst); 959682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall a->val_dont_use = v; 969682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall atomic_signal_fence(memory_order_seq_cst); 979682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall } 989682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall if (mo == memory_order_seq_cst) 999682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall atomic_thread_fence(memory_order_seq_cst); 1009682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall} 1019682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall 1029682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse HallINLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a, 1039682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall u32 v, memory_order mo) { 1049682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall (void)mo; 1059682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall DCHECK(!((uptr)a % sizeof(*a))); 1069682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall return (u32)_InterlockedExchangeAdd( 1079682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall (volatile long*)&a->val_dont_use, (long)v); // NOLINT 1089682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall} 1099682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall 1109682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse HallINLINE uptr atomic_fetch_add(volatile atomic_uintptr_t *a, 1119682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall uptr v, memory_order mo) { 1129682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall (void)mo; 1139682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall DCHECK(!((uptr)a % sizeof(*a))); 1149682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall#ifdef _WIN64 1159682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall return (uptr)_InterlockedExchangeAdd64( 1169682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall (volatile long long*)&a->val_dont_use, (long long)v); // NOLINT 1179682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall#else 1189682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall return (uptr)_InterlockedExchangeAdd( 1199682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall (volatile long*)&a->val_dont_use, (long)v); // NOLINT 1209682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall#endif 1219682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall} 1229682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall 1239682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse HallINLINE u32 atomic_fetch_sub(volatile atomic_uint32_t *a, 1249682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall u32 v, memory_order mo) { 1259682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall (void)mo; 1269682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall DCHECK(!((uptr)a % sizeof(*a))); 1279682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall return (u32)_InterlockedExchangeAdd( 1289682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall (volatile long*)&a->val_dont_use, -(long)v); // NOLINT 1299682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall} 1309682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall 1319682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse HallINLINE uptr atomic_fetch_sub(volatile atomic_uintptr_t *a, 1329682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall uptr v, memory_order mo) { 1339682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall (void)mo; 1349682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall DCHECK(!((uptr)a % sizeof(*a))); 1359682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall#ifdef _WIN64 1369682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall return (uptr)_InterlockedExchangeAdd64( 1379682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall (volatile long long*)&a->val_dont_use, -(long long)v); // NOLINT 1389682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall#else 1399682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall return (uptr)_InterlockedExchangeAdd( 1409682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall (volatile long*)&a->val_dont_use, -(long)v); // NOLINT 1419682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall#endif 1429682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall} 1439682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall 1449682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse HallINLINE u8 atomic_exchange(volatile atomic_uint8_t *a, 1459682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall u8 v, memory_order mo) { 1469682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall (void)mo; 1479682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall DCHECK(!((uptr)a % sizeof(*a))); 1489682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall __asm { 1499682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall mov eax, a 1509682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall mov cl, v 1519682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall xchg [eax], cl // NOLINT 1529682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall mov v, cl 1539682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall } 1549682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall return v; 1559682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall} 1569682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall 1579682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse HallINLINE u16 atomic_exchange(volatile atomic_uint16_t *a, 1589682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall u16 v, memory_order mo) { 1599682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall (void)mo; 1609682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall DCHECK(!((uptr)a % sizeof(*a))); 1619682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall __asm { 1629682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall mov eax, a 1639682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall mov cx, v 1649682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall xchg [eax], cx // NOLINT 1659682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall mov v, cx 1669682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall } 1679682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall return v; 1689682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall} 1699682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall 1709682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse HallINLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a, 1719682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall u8 *cmp, 1729682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall u8 xchgv, 1739682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall memory_order mo) { 1749682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall (void)mo; 1759682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall DCHECK(!((uptr)a % sizeof(*a))); 1769682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall u8 cmpv = *cmp; 1779682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall u8 prev; 1789682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall __asm { 1799682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall mov al, cmpv 1809682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall mov ecx, a 1819682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall mov dl, xchgv 1829682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall lock cmpxchg [ecx], dl 1839682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall mov prev, al 1849682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall } 1859682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall if (prev == cmpv) 1869682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall return true; 1879682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall *cmp = prev; 1889682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall return false; 1899682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall} 1909682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall 1919682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse HallINLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a, 1929682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall uptr *cmp, 1939682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall uptr xchg, 1949682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall memory_order mo) { 1959682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall uptr cmpv = *cmp; 1969682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall uptr prev = (uptr)_InterlockedCompareExchangePointer( 1979682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall (void*volatile*)&a->val_dont_use, (void*)xchg, (void*)cmpv); 1989682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall if (prev == cmpv) 1999682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall return true; 2009682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall *cmp = prev; 2019682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall return false; 2029682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall} 2039682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall 2049682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse HallINLINE bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a, 2059682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall u16 *cmp, 2069682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall u16 xchg, 2079682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall memory_order mo) { 2089682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall u16 cmpv = *cmp; 2099682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall u16 prev = (u16)_InterlockedCompareExchange16( 2109682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall (volatile short*)&a->val_dont_use, (short)xchg, (short)cmpv); 2119682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall if (prev == cmpv) 2129682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall return true; 2139682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall *cmp = prev; 2149682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall return false; 2159682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall} 2169682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall 2179682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse HallINLINE bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a, 2189682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall u32 *cmp, 2199682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall u32 xchg, 2209682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall memory_order mo) { 2219682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall u32 cmpv = *cmp; 2229682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall u32 prev = (u32)_InterlockedCompareExchange( 2239682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall (volatile long*)&a->val_dont_use, (long)xchg, (long)cmpv); 2249682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall if (prev == cmpv) 2259682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall return true; 2269682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall *cmp = prev; 2279682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall return false; 2289682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall} 2299682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall 2309682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse HallINLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a, 2319682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall u64 *cmp, 2329682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall u64 xchg, 2339682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall memory_order mo) { 2349682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall u64 cmpv = *cmp; 2359682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall u64 prev = (u64)_InterlockedCompareExchange64( 2369682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall (volatile long long*)&a->val_dont_use, (long long)xchg, (long long)cmpv); 2379682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall if (prev == cmpv) 2389682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall return true; 2399682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall *cmp = prev; 2409682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall return false; 2419682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall} 2429682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall 2439682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Halltemplate<typename T> 2449682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse HallINLINE bool atomic_compare_exchange_weak(volatile T *a, 2459682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall typename T::Type *cmp, 2469682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall typename T::Type xchg, 2479682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall memory_order mo) { 2489682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall return atomic_compare_exchange_strong(a, cmp, xchg, mo); 2499682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall} 2509682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall 2519682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall} // namespace __sanitizer 2529682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall 2539682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall#endif // SANITIZER_ATOMIC_CLANG_H 2549682c8870b8ff5e4ac2e4c70b759f791c6f38c1fJesse Hall