1/* ===-------- intrin.h ---------------------------------------------------=== 2 * 3 * Permission is hereby granted, free of charge, to any person obtaining a copy 4 * of this software and associated documentation files (the "Software"), to deal 5 * in the Software without restriction, including without limitation the rights 6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 * copies of the Software, and to permit persons to whom the Software is 8 * furnished to do so, subject to the following conditions: 9 * 10 * The above copyright notice and this permission notice shall be included in 11 * all copies or substantial portions of the Software. 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 * THE SOFTWARE. 20 * 21 *===-----------------------------------------------------------------------=== 22 */ 23 24/* Only include this if we're compiling for the windows platform. */ 25#ifndef _MSC_VER 26#include_next <intrin.h> 27#else 28 29#ifndef __INTRIN_H 30#define __INTRIN_H 31 32/* First include the standard intrinsics. */ 33#if defined(__i386__) || defined(__x86_64__) 34#include <x86intrin.h> 35#endif 36 37#if defined(__arm__) 38#include <armintr.h> 39#endif 40 41/* For the definition of jmp_buf. */ 42#if __STDC_HOSTED__ 43#include <setjmp.h> 44#endif 45 46/* Define the default attributes for the functions in this file. */ 47#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__)) 48 49#ifdef __cplusplus 50extern "C" { 51#endif 52 53#if defined(__MMX__) 54/* And the random ones that aren't in those files. */ 55__m64 _m_from_float(float); 56float _m_to_float(__m64); 57#endif 58 59/* Other assorted instruction intrinsics. */ 60void __addfsbyte(unsigned long, unsigned char); 61void __addfsdword(unsigned long, unsigned long); 62void __addfsword(unsigned long, unsigned short); 63void __code_seg(const char *); 64static __inline__ 65void __cpuid(int[4], int); 66static __inline__ 67void __cpuidex(int[4], int, int); 68void __debugbreak(void); 69static __inline__ 70__int64 __emul(int, int); 71static __inline__ 72unsigned __int64 __emulu(unsigned int, unsigned int); 73void __cdecl __fastfail(unsigned int); 74unsigned int __getcallerseflags(void); 75static __inline__ 76void __halt(void); 77unsigned char __inbyte(unsigned short); 78void __inbytestring(unsigned short, unsigned char *, unsigned long); 79void __incfsbyte(unsigned long); 80void __incfsdword(unsigned long); 81void __incfsword(unsigned long); 82unsigned long __indword(unsigned short); 83void __indwordstring(unsigned short, unsigned long *, unsigned long); 84void __int2c(void); 85void __invlpg(void *); 86unsigned short __inword(unsigned short); 87void __inwordstring(unsigned short, unsigned short *, unsigned long); 88void __lidt(void *); 89unsigned __int64 __ll_lshift(unsigned __int64, int); 90__int64 __ll_rshift(__int64, int); 91void __llwpcb(void *); 92unsigned char __lwpins32(unsigned int, unsigned int, unsigned int); 93void __lwpval32(unsigned int, unsigned int, unsigned int); 94unsigned int __lzcnt(unsigned int); 95unsigned short __lzcnt16(unsigned short); 96static __inline__ 97void __movsb(unsigned char *, unsigned char const *, size_t); 98static __inline__ 99void __movsd(unsigned long *, unsigned long const *, size_t); 100static __inline__ 101void __movsw(unsigned short *, unsigned short const *, size_t); 102static __inline__ 103void __nop(void); 104void __nvreg_restore_fence(void); 105void __nvreg_save_fence(void); 106void __outbyte(unsigned short, unsigned char); 107void __outbytestring(unsigned short, unsigned char *, unsigned long); 108void __outdword(unsigned short, unsigned long); 109void __outdwordstring(unsigned short, unsigned long *, unsigned long); 110void __outword(unsigned short, unsigned short); 111void __outwordstring(unsigned short, unsigned short *, unsigned long); 112static __inline__ 113unsigned int __popcnt(unsigned int); 114static __inline__ 115unsigned short __popcnt16(unsigned short); 116unsigned long __readcr0(void); 117unsigned long __readcr2(void); 118static __inline__ 119unsigned long __readcr3(void); 120unsigned long __readcr4(void); 121unsigned long __readcr8(void); 122unsigned int __readdr(unsigned int); 123#ifdef __i386__ 124static __inline__ 125unsigned char __readfsbyte(unsigned long); 126static __inline__ 127unsigned long __readfsdword(unsigned long); 128static __inline__ 129unsigned __int64 __readfsqword(unsigned long); 130static __inline__ 131unsigned short __readfsword(unsigned long); 132#endif 133static __inline__ 134unsigned __int64 __readmsr(unsigned long); 135unsigned __int64 __readpmc(unsigned long); 136unsigned long __segmentlimit(unsigned long); 137void __sidt(void *); 138void *__slwpcb(void); 139static __inline__ 140void __stosb(unsigned char *, unsigned char, size_t); 141static __inline__ 142void __stosd(unsigned long *, unsigned long, size_t); 143static __inline__ 144void __stosw(unsigned short *, unsigned short, size_t); 145void __svm_clgi(void); 146void __svm_invlpga(void *, int); 147void __svm_skinit(int); 148void __svm_stgi(void); 149void __svm_vmload(size_t); 150void __svm_vmrun(size_t); 151void __svm_vmsave(size_t); 152void __ud2(void); 153unsigned __int64 __ull_rshift(unsigned __int64, int); 154void __vmx_off(void); 155void __vmx_vmptrst(unsigned __int64 *); 156void __wbinvd(void); 157void __writecr0(unsigned int); 158static __inline__ 159void __writecr3(unsigned int); 160void __writecr4(unsigned int); 161void __writecr8(unsigned int); 162void __writedr(unsigned int, unsigned int); 163void __writefsbyte(unsigned long, unsigned char); 164void __writefsdword(unsigned long, unsigned long); 165void __writefsqword(unsigned long, unsigned __int64); 166void __writefsword(unsigned long, unsigned short); 167void __writemsr(unsigned long, unsigned __int64); 168static __inline__ 169void *_AddressOfReturnAddress(void); 170static __inline__ 171unsigned char _BitScanForward(unsigned long *_Index, unsigned long _Mask); 172static __inline__ 173unsigned char _BitScanReverse(unsigned long *_Index, unsigned long _Mask); 174static __inline__ 175unsigned char _bittest(long const *, long); 176static __inline__ 177unsigned char _bittestandcomplement(long *, long); 178static __inline__ 179unsigned char _bittestandreset(long *, long); 180static __inline__ 181unsigned char _bittestandset(long *, long); 182unsigned __int64 __cdecl _byteswap_uint64(unsigned __int64); 183unsigned long __cdecl _byteswap_ulong(unsigned long); 184unsigned short __cdecl _byteswap_ushort(unsigned short); 185void __cdecl _disable(void); 186void __cdecl _enable(void); 187long _InterlockedAddLargeStatistic(__int64 volatile *_Addend, long _Value); 188static __inline__ 189long _InterlockedAnd(long volatile *_Value, long _Mask); 190static __inline__ 191short _InterlockedAnd16(short volatile *_Value, short _Mask); 192static __inline__ 193char _InterlockedAnd8(char volatile *_Value, char _Mask); 194unsigned char _interlockedbittestandreset(long volatile *, long); 195static __inline__ 196unsigned char _interlockedbittestandset(long volatile *, long); 197static __inline__ 198long __cdecl _InterlockedCompareExchange(long volatile *_Destination, 199 long _Exchange, long _Comparand); 200long _InterlockedCompareExchange_HLEAcquire(long volatile *, long, long); 201long _InterlockedCompareExchange_HLERelease(long volatile *, long, long); 202static __inline__ 203short _InterlockedCompareExchange16(short volatile *_Destination, 204 short _Exchange, short _Comparand); 205static __inline__ 206__int64 _InterlockedCompareExchange64(__int64 volatile *_Destination, 207 __int64 _Exchange, __int64 _Comparand); 208__int64 _InterlockedcompareExchange64_HLEAcquire(__int64 volatile *, __int64, 209 __int64); 210__int64 _InterlockedCompareExchange64_HLERelease(__int64 volatile *, __int64, 211 __int64); 212static __inline__ 213char _InterlockedCompareExchange8(char volatile *_Destination, char _Exchange, 214 char _Comparand); 215void *_InterlockedCompareExchangePointer_HLEAcquire(void *volatile *, void *, 216 void *); 217void *_InterlockedCompareExchangePointer_HLERelease(void *volatile *, void *, 218 void *); 219static __inline__ 220long __cdecl _InterlockedDecrement(long volatile *_Addend); 221static __inline__ 222short _InterlockedDecrement16(short volatile *_Addend); 223long _InterlockedExchange(long volatile *_Target, long _Value); 224static __inline__ 225short _InterlockedExchange16(short volatile *_Target, short _Value); 226static __inline__ 227char _InterlockedExchange8(char volatile *_Target, char _Value); 228static __inline__ 229long __cdecl _InterlockedExchangeAdd(long volatile *_Addend, long _Value); 230long _InterlockedExchangeAdd_HLEAcquire(long volatile *, long); 231long _InterlockedExchangeAdd_HLERelease(long volatile *, long); 232static __inline__ 233short _InterlockedExchangeAdd16(short volatile *_Addend, short _Value); 234__int64 _InterlockedExchangeAdd64_HLEAcquire(__int64 volatile *, __int64); 235__int64 _InterlockedExchangeAdd64_HLERelease(__int64 volatile *, __int64); 236static __inline__ 237char _InterlockedExchangeAdd8(char volatile *_Addend, char _Value); 238static __inline__ 239long __cdecl _InterlockedIncrement(long volatile *_Addend); 240static __inline__ 241short _InterlockedIncrement16(short volatile *_Addend); 242static __inline__ 243long _InterlockedOr(long volatile *_Value, long _Mask); 244static __inline__ 245short _InterlockedOr16(short volatile *_Value, short _Mask); 246static __inline__ 247char _InterlockedOr8(char volatile *_Value, char _Mask); 248static __inline__ 249long _InterlockedXor(long volatile *_Value, long _Mask); 250static __inline__ 251short _InterlockedXor16(short volatile *_Value, short _Mask); 252static __inline__ 253char _InterlockedXor8(char volatile *_Value, char _Mask); 254void __cdecl _invpcid(unsigned int, void *); 255static __inline__ 256unsigned long __cdecl _lrotl(unsigned long, int); 257static __inline__ 258unsigned long __cdecl _lrotr(unsigned long, int); 259static __inline__ void 260__attribute__((__deprecated__("use other intrinsics or C++11 atomics instead"))) 261_ReadBarrier(void); 262static __inline__ void 263__attribute__((__deprecated__("use other intrinsics or C++11 atomics instead"))) 264_ReadWriteBarrier(void); 265static __inline__ 266void *_ReturnAddress(void); 267unsigned int _rorx_u32(unsigned int, const unsigned int); 268static __inline__ 269unsigned int __cdecl _rotl(unsigned int _Value, int _Shift); 270static __inline__ 271unsigned short _rotl16(unsigned short _Value, unsigned char _Shift); 272static __inline__ 273unsigned __int64 __cdecl _rotl64(unsigned __int64 _Value, int _Shift); 274static __inline__ 275unsigned char _rotl8(unsigned char _Value, unsigned char _Shift); 276static __inline__ 277unsigned int __cdecl _rotr(unsigned int _Value, int _Shift); 278static __inline__ 279unsigned short _rotr16(unsigned short _Value, unsigned char _Shift); 280static __inline__ 281unsigned __int64 __cdecl _rotr64(unsigned __int64 _Value, int _Shift); 282static __inline__ 283unsigned char _rotr8(unsigned char _Value, unsigned char _Shift); 284int _sarx_i32(int, unsigned int); 285#if __STDC_HOSTED__ 286int __cdecl _setjmp(jmp_buf); 287#endif 288unsigned int _shlx_u32(unsigned int, unsigned int); 289unsigned int _shrx_u32(unsigned int, unsigned int); 290void _Store_HLERelease(long volatile *, long); 291void _Store64_HLERelease(__int64 volatile *, __int64); 292void _StorePointer_HLERelease(void *volatile *, void *); 293static __inline__ void 294__attribute__((__deprecated__("use other intrinsics or C++11 atomics instead"))) 295_WriteBarrier(void); 296unsigned __int32 xbegin(void); 297void _xend(void); 298static __inline__ 299#define _XCR_XFEATURE_ENABLED_MASK 0 300unsigned __int64 __cdecl _xgetbv(unsigned int); 301void __cdecl _xsetbv(unsigned int, unsigned __int64); 302 303/* These additional intrinsics are turned on in x64/amd64/x86_64 mode. */ 304#ifdef __x86_64__ 305void __addgsbyte(unsigned long, unsigned char); 306void __addgsdword(unsigned long, unsigned long); 307void __addgsqword(unsigned long, unsigned __int64); 308void __addgsword(unsigned long, unsigned short); 309static __inline__ 310void __faststorefence(void); 311void __incgsbyte(unsigned long); 312void __incgsdword(unsigned long); 313void __incgsqword(unsigned long); 314void __incgsword(unsigned long); 315unsigned char __lwpins64(unsigned __int64, unsigned int, unsigned int); 316void __lwpval64(unsigned __int64, unsigned int, unsigned int); 317unsigned __int64 __lzcnt64(unsigned __int64); 318static __inline__ 319void __movsq(unsigned long long *, unsigned long long const *, size_t); 320static __inline__ 321unsigned __int64 __popcnt64(unsigned __int64); 322static __inline__ 323unsigned char __readgsbyte(unsigned long); 324static __inline__ 325unsigned long __readgsdword(unsigned long); 326static __inline__ 327unsigned __int64 __readgsqword(unsigned long); 328unsigned short __readgsword(unsigned long); 329unsigned __int64 __shiftleft128(unsigned __int64 _LowPart, 330 unsigned __int64 _HighPart, 331 unsigned char _Shift); 332unsigned __int64 __shiftright128(unsigned __int64 _LowPart, 333 unsigned __int64 _HighPart, 334 unsigned char _Shift); 335static __inline__ 336void __stosq(unsigned __int64 *, unsigned __int64, size_t); 337unsigned char __vmx_on(unsigned __int64 *); 338unsigned char __vmx_vmclear(unsigned __int64 *); 339unsigned char __vmx_vmlaunch(void); 340unsigned char __vmx_vmptrld(unsigned __int64 *); 341unsigned char __vmx_vmread(size_t, size_t *); 342unsigned char __vmx_vmresume(void); 343unsigned char __vmx_vmwrite(size_t, size_t); 344void __writegsbyte(unsigned long, unsigned char); 345void __writegsdword(unsigned long, unsigned long); 346void __writegsqword(unsigned long, unsigned __int64); 347void __writegsword(unsigned long, unsigned short); 348static __inline__ 349unsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask); 350static __inline__ 351unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask); 352static __inline__ 353unsigned char _bittest64(__int64 const *, __int64); 354static __inline__ 355unsigned char _bittestandcomplement64(__int64 *, __int64); 356static __inline__ 357unsigned char _bittestandreset64(__int64 *, __int64); 358static __inline__ 359unsigned char _bittestandset64(__int64 *, __int64); 360unsigned __int64 __cdecl _byteswap_uint64(unsigned __int64); 361long _InterlockedAnd_np(long volatile *_Value, long _Mask); 362short _InterlockedAnd16_np(short volatile *_Value, short _Mask); 363__int64 _InterlockedAnd64_np(__int64 volatile *_Value, __int64 _Mask); 364char _InterlockedAnd8_np(char volatile *_Value, char _Mask); 365unsigned char _interlockedbittestandreset64(__int64 volatile *, __int64); 366static __inline__ 367unsigned char _interlockedbittestandset64(__int64 volatile *, __int64); 368long _InterlockedCompareExchange_np(long volatile *_Destination, long _Exchange, 369 long _Comparand); 370unsigned char _InterlockedCompareExchange128(__int64 volatile *_Destination, 371 __int64 _ExchangeHigh, 372 __int64 _ExchangeLow, 373 __int64 *_CompareandResult); 374unsigned char _InterlockedCompareExchange128_np(__int64 volatile *_Destination, 375 __int64 _ExchangeHigh, 376 __int64 _ExchangeLow, 377 __int64 *_ComparandResult); 378short _InterlockedCompareExchange16_np(short volatile *_Destination, 379 short _Exchange, short _Comparand); 380__int64 _InterlockedCompareExchange64_HLEAcquire(__int64 volatile *, __int64, 381 __int64); 382__int64 _InterlockedCompareExchange64_HLERelease(__int64 volatile *, __int64, 383 __int64); 384__int64 _InterlockedCompareExchange64_np(__int64 volatile *_Destination, 385 __int64 _Exchange, __int64 _Comparand); 386void *_InterlockedCompareExchangePointer(void *volatile *_Destination, 387 void *_Exchange, void *_Comparand); 388void *_InterlockedCompareExchangePointer_np(void *volatile *_Destination, 389 void *_Exchange, void *_Comparand); 390void *_InterlockedExchangePointer(void *volatile *_Target, void *_Value); 391long _InterlockedOr_np(long volatile *_Value, long _Mask); 392short _InterlockedOr16_np(short volatile *_Value, short _Mask); 393__int64 _InterlockedOr64_np(__int64 volatile *_Value, __int64 _Mask); 394char _InterlockedOr8_np(char volatile *_Value, char _Mask); 395long _InterlockedXor_np(long volatile *_Value, long _Mask); 396short _InterlockedXor16_np(short volatile *_Value, short _Mask); 397__int64 _InterlockedXor64_np(__int64 volatile *_Value, __int64 _Mask); 398char _InterlockedXor8_np(char volatile *_Value, char _Mask); 399unsigned __int64 _rorx_u64(unsigned __int64, const unsigned int); 400__int64 _sarx_i64(__int64, unsigned int); 401#if __STDC_HOSTED__ 402int __cdecl _setjmpex(jmp_buf); 403#endif 404unsigned __int64 _shlx_u64(unsigned __int64, unsigned int); 405unsigned __int64 _shrx_u64(unsigned __int64, unsigned int); 406static __inline__ 407__int64 __mulh(__int64, __int64); 408static __inline__ 409unsigned __int64 __umulh(unsigned __int64, unsigned __int64); 410static __inline__ 411__int64 _mul128(__int64, __int64, __int64*); 412static __inline__ 413unsigned __int64 _umul128(unsigned __int64, 414 unsigned __int64, 415 unsigned __int64*); 416 417#endif /* __x86_64__ */ 418 419#if defined(__x86_64__) || defined(__arm__) 420 421static __inline__ 422__int64 _InterlockedDecrement64(__int64 volatile *_Addend); 423static __inline__ 424__int64 _InterlockedExchange64(__int64 volatile *_Target, __int64 _Value); 425static __inline__ 426__int64 _InterlockedExchangeAdd64(__int64 volatile *_Addend, __int64 _Value); 427static __inline__ 428__int64 _InterlockedExchangeSub64(__int64 volatile *_Subend, __int64 _Value); 429static __inline__ 430__int64 _InterlockedIncrement64(__int64 volatile *_Addend); 431static __inline__ 432__int64 _InterlockedOr64(__int64 volatile *_Value, __int64 _Mask); 433static __inline__ 434__int64 _InterlockedXor64(__int64 volatile *_Value, __int64 _Mask); 435static __inline__ 436__int64 _InterlockedAnd64(__int64 volatile *_Value, __int64 _Mask); 437 438#endif 439 440/*----------------------------------------------------------------------------*\ 441|* Bit Counting and Testing 442\*----------------------------------------------------------------------------*/ 443static __inline__ unsigned char __DEFAULT_FN_ATTRS 444_bittest(long const *_BitBase, long _BitPos) { 445 return (*_BitBase >> _BitPos) & 1; 446} 447static __inline__ unsigned char __DEFAULT_FN_ATTRS 448_bittestandcomplement(long *_BitBase, long _BitPos) { 449 unsigned char _Res = (*_BitBase >> _BitPos) & 1; 450 *_BitBase = *_BitBase ^ (1 << _BitPos); 451 return _Res; 452} 453static __inline__ unsigned char __DEFAULT_FN_ATTRS 454_bittestandreset(long *_BitBase, long _BitPos) { 455 unsigned char _Res = (*_BitBase >> _BitPos) & 1; 456 *_BitBase = *_BitBase & ~(1 << _BitPos); 457 return _Res; 458} 459static __inline__ unsigned char __DEFAULT_FN_ATTRS 460_bittestandset(long *_BitBase, long _BitPos) { 461 unsigned char _Res = (*_BitBase >> _BitPos) & 1; 462 *_BitBase = *_BitBase | (1 << _BitPos); 463 return _Res; 464} 465static __inline__ unsigned char __DEFAULT_FN_ATTRS 466_interlockedbittestandset(long volatile *_BitBase, long _BitPos) { 467 long _PrevVal = __atomic_fetch_or(_BitBase, 1l << _BitPos, __ATOMIC_SEQ_CST); 468 return (_PrevVal >> _BitPos) & 1; 469} 470#if defined(__arm__) || defined(__aarch64__) 471static __inline__ unsigned char __DEFAULT_FN_ATTRS 472_interlockedbittestandset_acq(long volatile *_BitBase, long _BitPos) { 473 long _PrevVal = __atomic_fetch_or(_BitBase, 1l << _BitPos, __ATOMIC_ACQUIRE); 474 return (_PrevVal >> _BitPos) & 1; 475} 476static __inline__ unsigned char __DEFAULT_FN_ATTRS 477_interlockedbittestandset_nf(long volatile *_BitBase, long _BitPos) { 478 long _PrevVal = __atomic_fetch_or(_BitBase, 1l << _BitPos, __ATOMIC_RELAXED); 479 return (_PrevVal >> _BitPos) & 1; 480} 481static __inline__ unsigned char __DEFAULT_FN_ATTRS 482_interlockedbittestandset_rel(long volatile *_BitBase, long _BitPos) { 483 long _PrevVal = __atomic_fetch_or(_BitBase, 1l << _BitPos, __ATOMIC_RELEASE); 484 return (_PrevVal >> _BitPos) & 1; 485} 486#endif 487#ifdef __x86_64__ 488static __inline__ unsigned char __DEFAULT_FN_ATTRS 489_bittest64(__int64 const *_BitBase, __int64 _BitPos) { 490 return (*_BitBase >> _BitPos) & 1; 491} 492static __inline__ unsigned char __DEFAULT_FN_ATTRS 493_bittestandcomplement64(__int64 *_BitBase, __int64 _BitPos) { 494 unsigned char _Res = (*_BitBase >> _BitPos) & 1; 495 *_BitBase = *_BitBase ^ (1ll << _BitPos); 496 return _Res; 497} 498static __inline__ unsigned char __DEFAULT_FN_ATTRS 499_bittestandreset64(__int64 *_BitBase, __int64 _BitPos) { 500 unsigned char _Res = (*_BitBase >> _BitPos) & 1; 501 *_BitBase = *_BitBase & ~(1ll << _BitPos); 502 return _Res; 503} 504static __inline__ unsigned char __DEFAULT_FN_ATTRS 505_bittestandset64(__int64 *_BitBase, __int64 _BitPos) { 506 unsigned char _Res = (*_BitBase >> _BitPos) & 1; 507 *_BitBase = *_BitBase | (1ll << _BitPos); 508 return _Res; 509} 510static __inline__ unsigned char __DEFAULT_FN_ATTRS 511_interlockedbittestandset64(__int64 volatile *_BitBase, __int64 _BitPos) { 512 long long _PrevVal = 513 __atomic_fetch_or(_BitBase, 1ll << _BitPos, __ATOMIC_SEQ_CST); 514 return (_PrevVal >> _BitPos) & 1; 515} 516#endif 517/*----------------------------------------------------------------------------*\ 518|* Interlocked Exchange Add 519\*----------------------------------------------------------------------------*/ 520#if defined(__arm__) || defined(__aarch64__) 521static __inline__ char __DEFAULT_FN_ATTRS 522_InterlockedExchangeAdd8_acq(char volatile *_Addend, char _Value) { 523 return __atomic_fetch_add(_Addend, _Value, __ATOMIC_ACQUIRE); 524} 525static __inline__ char __DEFAULT_FN_ATTRS 526_InterlockedExchangeAdd8_nf(char volatile *_Addend, char _Value) { 527 return __atomic_fetch_add(_Addend, _Value, __ATOMIC_RELAXED); 528} 529static __inline__ char __DEFAULT_FN_ATTRS 530_InterlockedExchangeAdd8_rel(char volatile *_Addend, char _Value) { 531 return __atomic_fetch_add(_Addend, _Value, __ATOMIC_RELAXED); 532} 533static __inline__ short __DEFAULT_FN_ATTRS 534_InterlockedExchangeAdd16_acq(short volatile *_Addend, short _Value) { 535 return __atomic_fetch_add(_Addend, _Value, __ATOMIC_ACQUIRE); 536} 537static __inline__ short __DEFAULT_FN_ATTRS 538_InterlockedExchangeAdd16_nf(short volatile *_Addend, short _Value) { 539 return __atomic_fetch_add(_Addend, _Value, __ATOMIC_RELAXED); 540} 541static __inline__ short __DEFAULT_FN_ATTRS 542_InterlockedExchangeAdd16_rel(short volatile *_Addend, short _Value) { 543 return __atomic_fetch_add(_Addend, _Value, __ATOMIC_RELEASE); 544} 545static __inline__ long __DEFAULT_FN_ATTRS 546_InterlockedExchangeAdd_acq(long volatile *_Addend, long _Value) { 547 return __atomic_fetch_add(_Addend, _Value, __ATOMIC_ACQUIRE); 548} 549static __inline__ long __DEFAULT_FN_ATTRS 550_InterlockedExchangeAdd_nf(long volatile *_Addend, long _Value) { 551 return __atomic_fetch_add(_Addend, _Value, __ATOMIC_RELAXED); 552} 553static __inline__ long __DEFAULT_FN_ATTRS 554_InterlockedExchangeAdd_rel(long volatile *_Addend, long _Value) { 555 return __atomic_fetch_add(_Addend, _Value, __ATOMIC_RELEASE); 556} 557static __inline__ __int64 __DEFAULT_FN_ATTRS 558_InterlockedExchangeAdd64_acq(__int64 volatile *_Addend, __int64 _Value) { 559 return __atomic_fetch_add(_Addend, _Value, __ATOMIC_ACQUIRE); 560} 561static __inline__ __int64 __DEFAULT_FN_ATTRS 562_InterlockedExchangeAdd64_nf(__int64 volatile *_Addend, __int64 _Value) { 563 return __atomic_fetch_add(_Addend, _Value, __ATOMIC_RELAXED); 564} 565static __inline__ __int64 __DEFAULT_FN_ATTRS 566_InterlockedExchangeAdd64_rel(__int64 volatile *_Addend, __int64 _Value) { 567 return __atomic_fetch_add(_Addend, _Value, __ATOMIC_RELEASE); 568} 569#endif 570/*----------------------------------------------------------------------------*\ 571|* Interlocked Increment 572\*----------------------------------------------------------------------------*/ 573#if defined(__arm__) || defined(__aarch64__) 574static __inline__ short __DEFAULT_FN_ATTRS 575_InterlockedIncrement16_acq(short volatile *_Value) { 576 return __atomic_add_fetch(_Value, 1, __ATOMIC_ACQUIRE); 577} 578static __inline__ short __DEFAULT_FN_ATTRS 579_InterlockedIncrement16_nf(short volatile *_Value) { 580 return __atomic_add_fetch(_Value, 1, __ATOMIC_RELAXED); 581} 582static __inline__ short __DEFAULT_FN_ATTRS 583_InterlockedIncrement16_rel(short volatile *_Value) { 584 return __atomic_add_fetch(_Value, 1, __ATOMIC_RELEASE); 585} 586static __inline__ long __DEFAULT_FN_ATTRS 587_InterlockedIncrement_acq(long volatile *_Value) { 588 return __atomic_add_fetch(_Value, 1, __ATOMIC_ACQUIRE); 589} 590static __inline__ long __DEFAULT_FN_ATTRS 591_InterlockedIncrement_nf(long volatile *_Value) { 592 return __atomic_add_fetch(_Value, 1, __ATOMIC_RELAXED); 593} 594static __inline__ long __DEFAULT_FN_ATTRS 595_InterlockedIncrement_rel(long volatile *_Value) { 596 return __atomic_add_fetch(_Value, 1, __ATOMIC_RELEASE); 597} 598static __inline__ __int64 __DEFAULT_FN_ATTRS 599_InterlockedIncrement64_acq(__int64 volatile *_Value) { 600 return __atomic_add_fetch(_Value, 1, __ATOMIC_ACQUIRE); 601} 602static __inline__ __int64 __DEFAULT_FN_ATTRS 603_InterlockedIncrement64_nf(__int64 volatile *_Value) { 604 return __atomic_add_fetch(_Value, 1, __ATOMIC_RELAXED); 605} 606static __inline__ __int64 __DEFAULT_FN_ATTRS 607_InterlockedIncrement64_rel(__int64 volatile *_Value) { 608 return __atomic_add_fetch(_Value, 1, __ATOMIC_RELEASE); 609} 610#endif 611/*----------------------------------------------------------------------------*\ 612|* Interlocked Decrement 613\*----------------------------------------------------------------------------*/ 614#if defined(__arm__) || defined(__aarch64__) 615static __inline__ short __DEFAULT_FN_ATTRS 616_InterlockedDecrement16_acq(short volatile *_Value) { 617 return __atomic_sub_fetch(_Value, 1, __ATOMIC_ACQUIRE); 618} 619static __inline__ short __DEFAULT_FN_ATTRS 620_InterlockedDecrement16_nf(short volatile *_Value) { 621 return __atomic_sub_fetch(_Value, 1, __ATOMIC_RELAXED); 622} 623static __inline__ short __DEFAULT_FN_ATTRS 624_InterlockedDecrement16_rel(short volatile *_Value) { 625 return __atomic_sub_fetch(_Value, 1, __ATOMIC_RELEASE); 626} 627static __inline__ long __DEFAULT_FN_ATTRS 628_InterlockedDecrement_acq(long volatile *_Value) { 629 return __atomic_sub_fetch(_Value, 1, __ATOMIC_ACQUIRE); 630} 631static __inline__ long __DEFAULT_FN_ATTRS 632_InterlockedDecrement_nf(long volatile *_Value) { 633 return __atomic_sub_fetch(_Value, 1, __ATOMIC_RELAXED); 634} 635static __inline__ long __DEFAULT_FN_ATTRS 636_InterlockedDecrement_rel(long volatile *_Value) { 637 return __atomic_sub_fetch(_Value, 1, __ATOMIC_RELEASE); 638} 639static __inline__ __int64 __DEFAULT_FN_ATTRS 640_InterlockedDecrement64_acq(__int64 volatile *_Value) { 641 return __atomic_sub_fetch(_Value, 1, __ATOMIC_ACQUIRE); 642} 643static __inline__ __int64 __DEFAULT_FN_ATTRS 644_InterlockedDecrement64_nf(__int64 volatile *_Value) { 645 return __atomic_sub_fetch(_Value, 1, __ATOMIC_RELAXED); 646} 647static __inline__ __int64 __DEFAULT_FN_ATTRS 648_InterlockedDecrement64_rel(__int64 volatile *_Value) { 649 return __atomic_sub_fetch(_Value, 1, __ATOMIC_RELEASE); 650} 651#endif 652/*----------------------------------------------------------------------------*\ 653|* Interlocked And 654\*----------------------------------------------------------------------------*/ 655#if defined(__arm__) || defined(__aarch64__) 656static __inline__ char __DEFAULT_FN_ATTRS 657_InterlockedAnd8_acq(char volatile *_Value, char _Mask) { 658 return __atomic_fetch_and(_Value, _Mask, __ATOMIC_ACQUIRE); 659} 660static __inline__ char __DEFAULT_FN_ATTRS 661_InterlockedAnd8_nf(char volatile *_Value, char _Mask) { 662 return __atomic_fetch_and(_Value, _Mask, __ATOMIC_RELAXED); 663} 664static __inline__ char __DEFAULT_FN_ATTRS 665_InterlockedAnd8_rel(char volatile *_Value, char _Mask) { 666 return __atomic_fetch_and(_Value, _Mask, __ATOMIC_RELEASE); 667} 668static __inline__ short __DEFAULT_FN_ATTRS 669_InterlockedAnd16_acq(short volatile *_Value, short _Mask) { 670 return __atomic_fetch_and(_Value, _Mask, __ATOMIC_ACQUIRE); 671} 672static __inline__ short __DEFAULT_FN_ATTRS 673_InterlockedAnd16_nf(short volatile *_Value, short _Mask) { 674 return __atomic_fetch_and(_Value, _Mask, __ATOMIC_RELAXED); 675} 676static __inline__ short __DEFAULT_FN_ATTRS 677_InterlockedAnd16_rel(short volatile *_Value, short _Mask) { 678 return __atomic_fetch_and(_Value, _Mask, __ATOMIC_RELEASE); 679} 680static __inline__ long __DEFAULT_FN_ATTRS 681_InterlockedAnd_acq(long volatile *_Value, long _Mask) { 682 return __atomic_fetch_and(_Value, _Mask, __ATOMIC_ACQUIRE); 683} 684static __inline__ long __DEFAULT_FN_ATTRS 685_InterlockedAnd_nf(long volatile *_Value, long _Mask) { 686 return __atomic_fetch_and(_Value, _Mask, __ATOMIC_RELAXED); 687} 688static __inline__ long __DEFAULT_FN_ATTRS 689_InterlockedAnd_rel(long volatile *_Value, long _Mask) { 690 return __atomic_fetch_and(_Value, _Mask, __ATOMIC_RELEASE); 691} 692static __inline__ __int64 __DEFAULT_FN_ATTRS 693_InterlockedAnd64_acq(__int64 volatile *_Value, __int64 _Mask) { 694 return __atomic_fetch_and(_Value, _Mask, __ATOMIC_ACQUIRE); 695} 696static __inline__ __int64 __DEFAULT_FN_ATTRS 697_InterlockedAnd64_nf(__int64 volatile *_Value, __int64 _Mask) { 698 return __atomic_fetch_and(_Value, _Mask, __ATOMIC_RELAXED); 699} 700static __inline__ __int64 __DEFAULT_FN_ATTRS 701_InterlockedAnd64_rel(__int64 volatile *_Value, __int64 _Mask) { 702 return __atomic_fetch_and(_Value, _Mask, __ATOMIC_RELEASE); 703} 704#endif 705/*----------------------------------------------------------------------------*\ 706|* Interlocked Or 707\*----------------------------------------------------------------------------*/ 708#if defined(__arm__) || defined(__aarch64__) 709static __inline__ char __DEFAULT_FN_ATTRS 710_InterlockedOr8_acq(char volatile *_Value, char _Mask) { 711 return __atomic_fetch_or(_Value, _Mask, __ATOMIC_ACQUIRE); 712} 713static __inline__ char __DEFAULT_FN_ATTRS 714_InterlockedOr8_nf(char volatile *_Value, char _Mask) { 715 return __atomic_fetch_or(_Value, _Mask, __ATOMIC_RELAXED); 716} 717static __inline__ char __DEFAULT_FN_ATTRS 718_InterlockedOr8_rel(char volatile *_Value, char _Mask) { 719 return __atomic_fetch_or(_Value, _Mask, __ATOMIC_RELEASE); 720} 721static __inline__ short __DEFAULT_FN_ATTRS 722_InterlockedOr16_acq(short volatile *_Value, short _Mask) { 723 return __atomic_fetch_or(_Value, _Mask, __ATOMIC_ACQUIRE); 724} 725static __inline__ short __DEFAULT_FN_ATTRS 726_InterlockedOr16_nf(short volatile *_Value, short _Mask) { 727 return __atomic_fetch_or(_Value, _Mask, __ATOMIC_RELAXED); 728} 729static __inline__ short __DEFAULT_FN_ATTRS 730_InterlockedOr16_rel(short volatile *_Value, short _Mask) { 731 return __atomic_fetch_or(_Value, _Mask, __ATOMIC_RELEASE); 732} 733static __inline__ long __DEFAULT_FN_ATTRS 734_InterlockedOr_acq(long volatile *_Value, long _Mask) { 735 return __atomic_fetch_or(_Value, _Mask, __ATOMIC_ACQUIRE); 736} 737static __inline__ long __DEFAULT_FN_ATTRS 738_InterlockedOr_nf(long volatile *_Value, long _Mask) { 739 return __atomic_fetch_or(_Value, _Mask, __ATOMIC_RELAXED); 740} 741static __inline__ long __DEFAULT_FN_ATTRS 742_InterlockedOr_rel(long volatile *_Value, long _Mask) { 743 return __atomic_fetch_or(_Value, _Mask, __ATOMIC_RELEASE); 744} 745static __inline__ __int64 __DEFAULT_FN_ATTRS 746_InterlockedOr64_acq(__int64 volatile *_Value, __int64 _Mask) { 747 return __atomic_fetch_or(_Value, _Mask, __ATOMIC_ACQUIRE); 748} 749static __inline__ __int64 __DEFAULT_FN_ATTRS 750_InterlockedOr64_nf(__int64 volatile *_Value, __int64 _Mask) { 751 return __atomic_fetch_or(_Value, _Mask, __ATOMIC_RELAXED); 752} 753static __inline__ __int64 __DEFAULT_FN_ATTRS 754_InterlockedOr64_rel(__int64 volatile *_Value, __int64 _Mask) { 755 return __atomic_fetch_or(_Value, _Mask, __ATOMIC_RELEASE); 756} 757#endif 758/*----------------------------------------------------------------------------*\ 759|* Interlocked Xor 760\*----------------------------------------------------------------------------*/ 761#if defined(__arm__) || defined(__aarch64__) 762static __inline__ char __DEFAULT_FN_ATTRS 763_InterlockedXor8_acq(char volatile *_Value, char _Mask) { 764 return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_ACQUIRE); 765} 766static __inline__ char __DEFAULT_FN_ATTRS 767_InterlockedXor8_nf(char volatile *_Value, char _Mask) { 768 return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_RELAXED); 769} 770static __inline__ char __DEFAULT_FN_ATTRS 771_InterlockedXor8_rel(char volatile *_Value, char _Mask) { 772 return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_RELEASE); 773} 774static __inline__ short __DEFAULT_FN_ATTRS 775_InterlockedXor16_acq(short volatile *_Value, short _Mask) { 776 return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_ACQUIRE); 777} 778static __inline__ short __DEFAULT_FN_ATTRS 779_InterlockedXor16_nf(short volatile *_Value, short _Mask) { 780 return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_RELAXED); 781} 782static __inline__ short __DEFAULT_FN_ATTRS 783_InterlockedXor16_rel(short volatile *_Value, short _Mask) { 784 return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_RELEASE); 785} 786static __inline__ long __DEFAULT_FN_ATTRS 787_InterlockedXor_acq(long volatile *_Value, long _Mask) { 788 return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_ACQUIRE); 789} 790static __inline__ long __DEFAULT_FN_ATTRS 791_InterlockedXor_nf(long volatile *_Value, long _Mask) { 792 return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_RELAXED); 793} 794static __inline__ long __DEFAULT_FN_ATTRS 795_InterlockedXor_rel(long volatile *_Value, long _Mask) { 796 return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_RELEASE); 797} 798static __inline__ __int64 __DEFAULT_FN_ATTRS 799_InterlockedXor64_acq(__int64 volatile *_Value, __int64 _Mask) { 800 return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_ACQUIRE); 801} 802static __inline__ __int64 __DEFAULT_FN_ATTRS 803_InterlockedXor64_nf(__int64 volatile *_Value, __int64 _Mask) { 804 return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_RELAXED); 805} 806static __inline__ __int64 __DEFAULT_FN_ATTRS 807_InterlockedXor64_rel(__int64 volatile *_Value, __int64 _Mask) { 808 return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_RELEASE); 809} 810#endif 811/*----------------------------------------------------------------------------*\ 812|* Interlocked Exchange 813\*----------------------------------------------------------------------------*/ 814#if defined(__arm__) || defined(__aarch64__) 815static __inline__ char __DEFAULT_FN_ATTRS 816_InterlockedExchange8_acq(char volatile *_Target, char _Value) { 817 __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_ACQUIRE); 818 return _Value; 819} 820static __inline__ char __DEFAULT_FN_ATTRS 821_InterlockedExchange8_nf(char volatile *_Target, char _Value) { 822 __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_RELAXED); 823 return _Value; 824} 825static __inline__ char __DEFAULT_FN_ATTRS 826_InterlockedExchange8_rel(char volatile *_Target, char _Value) { 827 __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_RELEASE); 828 return _Value; 829} 830static __inline__ short __DEFAULT_FN_ATTRS 831_InterlockedExchange16_acq(short volatile *_Target, short _Value) { 832 __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_ACQUIRE); 833 return _Value; 834} 835static __inline__ short __DEFAULT_FN_ATTRS 836_InterlockedExchange16_nf(short volatile *_Target, short _Value) { 837 __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_RELAXED); 838 return _Value; 839} 840static __inline__ short __DEFAULT_FN_ATTRS 841_InterlockedExchange16_rel(short volatile *_Target, short _Value) { 842 __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_RELEASE); 843 return _Value; 844} 845static __inline__ long __DEFAULT_FN_ATTRS 846_InterlockedExchange_acq(long volatile *_Target, long _Value) { 847 __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_ACQUIRE); 848 return _Value; 849} 850static __inline__ long __DEFAULT_FN_ATTRS 851_InterlockedExchange_nf(long volatile *_Target, long _Value) { 852 __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_RELAXED); 853 return _Value; 854} 855static __inline__ long __DEFAULT_FN_ATTRS 856_InterlockedExchange_rel(long volatile *_Target, long _Value) { 857 __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_RELEASE); 858 return _Value; 859} 860static __inline__ __int64 __DEFAULT_FN_ATTRS 861_InterlockedExchange64_acq(__int64 volatile *_Target, __int64 _Value) { 862 __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_ACQUIRE); 863 return _Value; 864} 865static __inline__ __int64 __DEFAULT_FN_ATTRS 866_InterlockedExchange64_nf(__int64 volatile *_Target, __int64 _Value) { 867 __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_RELAXED); 868 return _Value; 869} 870static __inline__ __int64 __DEFAULT_FN_ATTRS 871_InterlockedExchange64_rel(__int64 volatile *_Target, __int64 _Value) { 872 __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_RELEASE); 873 return _Value; 874} 875#endif 876/*----------------------------------------------------------------------------*\ 877|* Interlocked Compare Exchange 878\*----------------------------------------------------------------------------*/ 879#if defined(__arm__) || defined(__aarch64__) 880static __inline__ char __DEFAULT_FN_ATTRS 881_InterlockedCompareExchange8_acq(char volatile *_Destination, 882 char _Exchange, char _Comparand) { 883 __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, 884 __ATOMIC_SEQ_CST, __ATOMIC_ACQUIRE); 885 return _Comparand; 886} 887static __inline__ char __DEFAULT_FN_ATTRS 888_InterlockedCompareExchange8_nf(char volatile *_Destination, 889 char _Exchange, char _Comparand) { 890 __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, 891 __ATOMIC_SEQ_CST, __ATOMIC_RELAXED); 892 return _Comparand; 893} 894static __inline__ char __DEFAULT_FN_ATTRS 895_InterlockedCompareExchange8_rel(char volatile *_Destination, 896 char _Exchange, char _Comparand) { 897 __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, 898 __ATOMIC_SEQ_CST, __ATOMIC_RELEASE); 899 return _Comparand; 900} 901static __inline__ short __DEFAULT_FN_ATTRS 902_InterlockedCompareExchange16_acq(short volatile *_Destination, 903 short _Exchange, short _Comparand) { 904 __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, 905 __ATOMIC_SEQ_CST, __ATOMIC_ACQUIRE); 906 return _Comparand; 907} 908static __inline__ short __DEFAULT_FN_ATTRS 909_InterlockedCompareExchange16_nf(short volatile *_Destination, 910 short _Exchange, short _Comparand) { 911 __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, 912 __ATOMIC_SEQ_CST, __ATOMIC_RELAXED); 913 return _Comparand; 914} 915static __inline__ short __DEFAULT_FN_ATTRS 916_InterlockedCompareExchange16_rel(short volatile *_Destination, 917 short _Exchange, short _Comparand) { 918 __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, 919 __ATOMIC_SEQ_CST, __ATOMIC_RELEASE); 920 return _Comparand; 921} 922static __inline__ long __DEFAULT_FN_ATTRS 923_InterlockedCompareExchange_acq(long volatile *_Destination, 924 long _Exchange, long _Comparand) { 925 __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, 926 __ATOMIC_SEQ_CST, __ATOMIC_ACQUIRE); 927 return _Comparand; 928} 929static __inline__ long __DEFAULT_FN_ATTRS 930_InterlockedCompareExchange_nf(long volatile *_Destination, 931 long _Exchange, long _Comparand) { 932 __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, 933 __ATOMIC_SEQ_CST, __ATOMIC_RELAXED); 934 return _Comparand; 935} 936static __inline__ short __DEFAULT_FN_ATTRS 937_InterlockedCompareExchange_rel(long volatile *_Destination, 938 long _Exchange, long _Comparand) { 939 __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, 940 __ATOMIC_SEQ_CST, __ATOMIC_RELEASE); 941 return _Comparand; 942} 943static __inline__ __int64 __DEFAULT_FN_ATTRS 944_InterlockedCompareExchange64_acq(__int64 volatile *_Destination, 945 __int64 _Exchange, __int64 _Comparand) { 946 __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, 947 __ATOMIC_SEQ_CST, __ATOMIC_ACQUIRE); 948 return _Comparand; 949} 950static __inline__ __int64 __DEFAULT_FN_ATTRS 951_InterlockedCompareExchange64_nf(__int64 volatile *_Destination, 952 __int64 _Exchange, __int64 _Comparand) { 953 __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, 954 __ATOMIC_SEQ_CST, __ATOMIC_RELAXED); 955 return _Comparand; 956} 957static __inline__ __int64 __DEFAULT_FN_ATTRS 958_InterlockedCompareExchange64_rel(__int64 volatile *_Destination, 959 __int64 _Exchange, __int64 _Comparand) { 960 __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, 961 __ATOMIC_SEQ_CST, __ATOMIC_RELEASE); 962 return _Comparand; 963} 964#endif 965/*----------------------------------------------------------------------------*\ 966|* readfs, readgs 967|* (Pointers in address space #256 and #257 are relative to the GS and FS 968|* segment registers, respectively.) 969\*----------------------------------------------------------------------------*/ 970#define __ptr_to_addr_space(__addr_space_nbr, __type, __offset) \ 971 ((volatile __type __attribute__((__address_space__(__addr_space_nbr)))*) \ 972 (__offset)) 973 974#ifdef __i386__ 975static __inline__ unsigned char __DEFAULT_FN_ATTRS 976__readfsbyte(unsigned long __offset) { 977 return *__ptr_to_addr_space(257, unsigned char, __offset); 978} 979static __inline__ unsigned short __DEFAULT_FN_ATTRS 980__readfsword(unsigned long __offset) { 981 return *__ptr_to_addr_space(257, unsigned short, __offset); 982} 983static __inline__ unsigned __int64 __DEFAULT_FN_ATTRS 984__readfsqword(unsigned long __offset) { 985 return *__ptr_to_addr_space(257, unsigned __int64, __offset); 986} 987#endif 988#ifdef __x86_64__ 989static __inline__ unsigned char __DEFAULT_FN_ATTRS 990__readgsbyte(unsigned long __offset) { 991 return *__ptr_to_addr_space(256, unsigned char, __offset); 992} 993static __inline__ unsigned short __DEFAULT_FN_ATTRS 994__readgsword(unsigned long __offset) { 995 return *__ptr_to_addr_space(256, unsigned short, __offset); 996} 997static __inline__ unsigned long __DEFAULT_FN_ATTRS 998__readgsdword(unsigned long __offset) { 999 return *__ptr_to_addr_space(256, unsigned long, __offset); 1000} 1001static __inline__ unsigned __int64 __DEFAULT_FN_ATTRS 1002__readgsqword(unsigned long __offset) { 1003 return *__ptr_to_addr_space(256, unsigned __int64, __offset); 1004} 1005#endif 1006#undef __ptr_to_addr_space 1007/*----------------------------------------------------------------------------*\ 1008|* movs, stos 1009\*----------------------------------------------------------------------------*/ 1010#if defined(__i386__) || defined(__x86_64__) 1011static __inline__ void __DEFAULT_FN_ATTRS 1012__movsb(unsigned char *__dst, unsigned char const *__src, size_t __n) { 1013 __asm__("rep movsb" : : "D"(__dst), "S"(__src), "c"(__n) 1014 : "%edi", "%esi", "%ecx"); 1015} 1016static __inline__ void __DEFAULT_FN_ATTRS 1017__movsd(unsigned long *__dst, unsigned long const *__src, size_t __n) { 1018 __asm__("rep movsl" : : "D"(__dst), "S"(__src), "c"(__n) 1019 : "%edi", "%esi", "%ecx"); 1020} 1021static __inline__ void __DEFAULT_FN_ATTRS 1022__movsw(unsigned short *__dst, unsigned short const *__src, size_t __n) { 1023 __asm__("rep movsw" : : "D"(__dst), "S"(__src), "c"(__n) 1024 : "%edi", "%esi", "%ecx"); 1025} 1026static __inline__ void __DEFAULT_FN_ATTRS 1027__stosd(unsigned long *__dst, unsigned long __x, size_t __n) { 1028 __asm__("rep stosl" : : "D"(__dst), "a"(__x), "c"(__n) 1029 : "%edi", "%ecx"); 1030} 1031static __inline__ void __DEFAULT_FN_ATTRS 1032__stosw(unsigned short *__dst, unsigned short __x, size_t __n) { 1033 __asm__("rep stosw" : : "D"(__dst), "a"(__x), "c"(__n) 1034 : "%edi", "%ecx"); 1035} 1036#endif 1037#ifdef __x86_64__ 1038static __inline__ void __DEFAULT_FN_ATTRS 1039__movsq(unsigned long long *__dst, unsigned long long const *__src, size_t __n) { 1040 __asm__("rep movsq" : : "D"(__dst), "S"(__src), "c"(__n) 1041 : "%edi", "%esi", "%ecx"); 1042} 1043static __inline__ void __DEFAULT_FN_ATTRS 1044__stosq(unsigned __int64 *__dst, unsigned __int64 __x, size_t __n) { 1045 __asm__("rep stosq" : : "D"(__dst), "a"(__x), "c"(__n) 1046 : "%edi", "%ecx"); 1047} 1048#endif 1049 1050/*----------------------------------------------------------------------------*\ 1051|* Misc 1052\*----------------------------------------------------------------------------*/ 1053#if defined(__i386__) || defined(__x86_64__) 1054static __inline__ void __DEFAULT_FN_ATTRS 1055__cpuid(int __info[4], int __level) { 1056 __asm__ ("cpuid" : "=a"(__info[0]), "=b" (__info[1]), "=c"(__info[2]), "=d"(__info[3]) 1057 : "a"(__level)); 1058} 1059static __inline__ void __DEFAULT_FN_ATTRS 1060__cpuidex(int __info[4], int __level, int __ecx) { 1061 __asm__ ("cpuid" : "=a"(__info[0]), "=b" (__info[1]), "=c"(__info[2]), "=d"(__info[3]) 1062 : "a"(__level), "c"(__ecx)); 1063} 1064static __inline__ unsigned __int64 __cdecl __DEFAULT_FN_ATTRS 1065_xgetbv(unsigned int __xcr_no) { 1066 unsigned int __eax, __edx; 1067 __asm__ ("xgetbv" : "=a" (__eax), "=d" (__edx) : "c" (__xcr_no)); 1068 return ((unsigned __int64)__edx << 32) | __eax; 1069} 1070static __inline__ void __DEFAULT_FN_ATTRS 1071__halt(void) { 1072 __asm__ volatile ("hlt"); 1073} 1074static __inline__ void __DEFAULT_FN_ATTRS 1075__nop(void) { 1076 __asm__ volatile ("nop"); 1077} 1078#endif 1079 1080/*----------------------------------------------------------------------------*\ 1081|* Privileged intrinsics 1082\*----------------------------------------------------------------------------*/ 1083#if defined(__i386__) || defined(__x86_64__) 1084static __inline__ unsigned __int64 __DEFAULT_FN_ATTRS 1085__readmsr(unsigned long __register) { 1086 // Loads the contents of a 64-bit model specific register (MSR) specified in 1087 // the ECX register into registers EDX:EAX. The EDX register is loaded with 1088 // the high-order 32 bits of the MSR and the EAX register is loaded with the 1089 // low-order 32 bits. If less than 64 bits are implemented in the MSR being 1090 // read, the values returned to EDX:EAX in unimplemented bit locations are 1091 // undefined. 1092 unsigned long __edx; 1093 unsigned long __eax; 1094 __asm__ ("rdmsr" : "=d"(__edx), "=a"(__eax) : "c"(__register)); 1095 return (((unsigned __int64)__edx) << 32) | (unsigned __int64)__eax; 1096} 1097 1098static __inline__ unsigned long __DEFAULT_FN_ATTRS 1099__readcr3(void) { 1100 unsigned long __cr3_val; 1101 __asm__ __volatile__ ("mov %%cr3, %0" : "=q"(__cr3_val) : : "memory"); 1102 return __cr3_val; 1103} 1104 1105static __inline__ void __DEFAULT_FN_ATTRS 1106__writecr3(unsigned int __cr3_val) { 1107 __asm__ ("mov %0, %%cr3" : : "q"(__cr3_val) : "memory"); 1108} 1109#endif 1110 1111#ifdef __cplusplus 1112} 1113#endif 1114 1115#undef __DEFAULT_FN_ATTRS 1116 1117#endif /* __INTRIN_H */ 1118#endif /* _MSC_VER */ 1119