14967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar/* ===-------- intrin.h ---------------------------------------------------=== 2f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher * 3f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher * Permission is hereby granted, free of charge, to any person obtaining a copy 4f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher * of this software and associated documentation files (the "Software"), to deal 5f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher * in the Software without restriction, including without limitation the rights 6f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher * copies of the Software, and to permit persons to whom the Software is 8f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher * furnished to do so, subject to the following conditions: 9f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher * 10f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher * The above copyright notice and this permission notice shall be included in 11f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher * all copies or substantial portions of the Software. 12f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher * 13f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher * THE SOFTWARE. 20f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher * 21f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher *===-----------------------------------------------------------------------=== 22f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher */ 23f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher 24f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher/* Only include this if we're compiling for the windows platform. */ 25f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher#ifndef _MSC_VER 264967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar#include_next <intrin.h> 27f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher#else 28f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher 29f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher#ifndef __INTRIN_H 30f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher#define __INTRIN_H 31f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher 32f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher/* First include the standard intrinsics. */ 33c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines#if defined(__i386__) || defined(__x86_64__) 34f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher#include <x86intrin.h> 35c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines#endif 36f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher 37651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines/* For the definition of jmp_buf. */ 38c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines#if __STDC_HOSTED__ 39651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#include <setjmp.h> 40c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines#endif 41651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar/* Define the default attributes for the functions in this file. */ 4387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__)) 4487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 451fe5ccfaebef277418b0ae44c52d7acf8dea0b0aEric Christopher#ifdef __cplusplus 461fe5ccfaebef277418b0ae44c52d7acf8dea0b0aEric Christopherextern "C" { 471fe5ccfaebef277418b0ae44c52d7acf8dea0b0aEric Christopher#endif 481fe5ccfaebef277418b0ae44c52d7acf8dea0b0aEric Christopher 49c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines#if defined(__MMX__) 50f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher/* And the random ones that aren't in those files. */ 51f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher__m64 _m_from_float(float); 52f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherfloat _m_to_float(__m64); 53c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines#endif 54f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher 55f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher/* Other assorted instruction intrinsics. */ 56f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __addfsbyte(unsigned long, unsigned char); 57f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __addfsdword(unsigned long, unsigned long); 58f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __addfsword(unsigned long, unsigned short); 59f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __code_seg(const char *); 60651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 61f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __cpuid(int[4], int); 62651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 63f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __cpuidex(int[4], int, int); 641fe5ccfaebef277418b0ae44c52d7acf8dea0b0aEric Christophervoid __debugbreak(void); 65f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher__int64 __emul(int, int); 66f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned __int64 __emulu(unsigned int, unsigned int); 671fe5ccfaebef277418b0ae44c52d7acf8dea0b0aEric Christophervoid __cdecl __fastfail(unsigned int); 68f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned int __getcallerseflags(void); 69651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 70f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __halt(void); 71f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned char __inbyte(unsigned short); 72f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __inbytestring(unsigned short, unsigned char *, unsigned long); 73f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __incfsbyte(unsigned long); 74f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __incfsdword(unsigned long); 75f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __incfsword(unsigned long); 76f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned long __indword(unsigned short); 77f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __indwordstring(unsigned short, unsigned long *, unsigned long); 78f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __int2c(void); 79f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __invlpg(void *); 80f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned short __inword(unsigned short); 81f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __inwordstring(unsigned short, unsigned short *, unsigned long); 82f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __lidt(void *); 83f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned __int64 __ll_lshift(unsigned __int64, int); 84f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher__int64 __ll_rshift(__int64, int); 85f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __llwpcb(void *); 86f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned char __lwpins32(unsigned int, unsigned int, unsigned int); 87f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __lwpval32(unsigned int, unsigned int, unsigned int); 88f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned int __lzcnt(unsigned int); 89f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned short __lzcnt16(unsigned short); 90651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 91f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __movsb(unsigned char *, unsigned char const *, size_t); 92651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 93f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __movsd(unsigned long *, unsigned long const *, size_t); 94651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 95f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __movsw(unsigned short *, unsigned short const *, size_t); 96f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __nop(void); 97f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __nvreg_restore_fence(void); 98f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __nvreg_save_fence(void); 99f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __outbyte(unsigned short, unsigned char); 100f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __outbytestring(unsigned short, unsigned char *, unsigned long); 101f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __outdword(unsigned short, unsigned long); 102f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __outdwordstring(unsigned short, unsigned long *, unsigned long); 103f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __outword(unsigned short, unsigned short); 104f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __outwordstring(unsigned short, unsigned short *, unsigned long); 105fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 106f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned int __popcnt(unsigned int); 107fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 108f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned short __popcnt16(unsigned short); 109f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned long __readcr0(void); 110f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned long __readcr2(void); 1116bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hinesstatic __inline__ 112f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned long __readcr3(void); 113651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesunsigned long __readcr4(void); 114f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned long __readcr8(void); 115f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned int __readdr(unsigned int); 116651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#ifdef __i386__ 117651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 118f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned char __readfsbyte(unsigned long); 119651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 120f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned long __readfsdword(unsigned long); 121651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 122f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned __int64 __readfsqword(unsigned long); 123651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 124f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned short __readfsword(unsigned long); 125651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#endif 1266bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hinesstatic __inline__ 127f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned __int64 __readmsr(unsigned long); 128f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned __int64 __readpmc(unsigned long); 129f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned long __segmentlimit(unsigned long); 130f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __sidt(void *); 131f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid *__slwpcb(void); 132651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 133f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __stosb(unsigned char *, unsigned char, size_t); 134651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 135f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __stosd(unsigned long *, unsigned long, size_t); 136651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 137f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __stosw(unsigned short *, unsigned short, size_t); 138f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __svm_clgi(void); 139f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __svm_invlpga(void *, int); 140f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __svm_skinit(int); 141f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __svm_stgi(void); 142f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __svm_vmload(size_t); 143f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __svm_vmrun(size_t); 144f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __svm_vmsave(size_t); 145f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __ud2(void); 146f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned __int64 __ull_rshift(unsigned __int64, int); 147f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __vmx_off(void); 148f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __vmx_vmptrst(unsigned __int64 *); 149f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __wbinvd(void); 150f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __writecr0(unsigned int); 1516bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hinesstatic __inline__ 152f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __writecr3(unsigned int); 153f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __writecr4(unsigned int); 154f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __writecr8(unsigned int); 155f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __writedr(unsigned int, unsigned int); 156f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __writefsbyte(unsigned long, unsigned char); 157f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __writefsdword(unsigned long, unsigned long); 158f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __writefsqword(unsigned long, unsigned __int64); 159f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __writefsword(unsigned long, unsigned short); 160f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __writemsr(unsigned long, unsigned __int64); 161fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 162f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid *_AddressOfReturnAddress(void); 163fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 164f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned char _BitScanForward(unsigned long *_Index, unsigned long _Mask); 165fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 166f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned char _BitScanReverse(unsigned long *_Index, unsigned long _Mask); 167fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 168f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned char _bittest(long const *, long); 169fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 170f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned char _bittestandcomplement(long *, long); 171fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 172f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned char _bittestandreset(long *, long); 173fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 174f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned char _bittestandset(long *, long); 175f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned __int64 __cdecl _byteswap_uint64(unsigned __int64); 176f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned long __cdecl _byteswap_ulong(unsigned long); 177f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned short __cdecl _byteswap_ushort(unsigned short); 178f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __cdecl _disable(void); 179f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __cdecl _enable(void); 180f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherlong _InterlockedAddLargeStatistic(__int64 volatile *_Addend, long _Value); 181fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 182f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherlong _InterlockedAnd(long volatile *_Value, long _Mask); 183fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 184f8e56745f5726a8f36444fe7e826e420340544bfEric Christophershort _InterlockedAnd16(short volatile *_Value, short _Mask); 185fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 186f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherchar _InterlockedAnd8(char volatile *_Value, char _Mask); 187f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned char _interlockedbittestandreset(long volatile *, long); 188651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 189f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned char _interlockedbittestandset(long volatile *, long); 190fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 191f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherlong __cdecl _InterlockedCompareExchange(long volatile *_Destination, 192f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher long _Exchange, long _Comparand); 193f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherlong _InterlockedCompareExchange_HLEAcquire(long volatile *, long, long); 194f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherlong _InterlockedCompareExchange_HLERelease(long volatile *, long, long); 195fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 196f8e56745f5726a8f36444fe7e826e420340544bfEric Christophershort _InterlockedCompareExchange16(short volatile *_Destination, 197f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher short _Exchange, short _Comparand); 198fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 199f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher__int64 _InterlockedCompareExchange64(__int64 volatile *_Destination, 200f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher __int64 _Exchange, __int64 _Comparand); 201f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher__int64 _InterlockedcompareExchange64_HLEAcquire(__int64 volatile *, __int64, 202f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher __int64); 203f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher__int64 _InterlockedCompareExchange64_HLERelease(__int64 volatile *, __int64, 204f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher __int64); 205fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 206f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherchar _InterlockedCompareExchange8(char volatile *_Destination, char _Exchange, 207f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher char _Comparand); 208f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid *_InterlockedCompareExchangePointer_HLEAcquire(void *volatile *, void *, 209f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher void *); 210f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid *_InterlockedCompareExchangePointer_HLERelease(void *volatile *, void *, 211f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher void *); 212fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 213f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherlong __cdecl _InterlockedDecrement(long volatile *_Addend); 214fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 215f8e56745f5726a8f36444fe7e826e420340544bfEric Christophershort _InterlockedDecrement16(short volatile *_Addend); 216c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hineslong _InterlockedExchange(long volatile *_Target, long _Value); 217fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 218f8e56745f5726a8f36444fe7e826e420340544bfEric Christophershort _InterlockedExchange16(short volatile *_Target, short _Value); 219fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 220f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherchar _InterlockedExchange8(char volatile *_Target, char _Value); 221fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 222f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherlong __cdecl _InterlockedExchangeAdd(long volatile *_Addend, long _Value); 223f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherlong _InterlockedExchangeAdd_HLEAcquire(long volatile *, long); 224f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherlong _InterlockedExchangeAdd_HLERelease(long volatile *, long); 225fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 226651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesshort _InterlockedExchangeAdd16(short volatile *_Addend, short _Value); 227651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__int64 _InterlockedExchangeAdd64_HLEAcquire(__int64 volatile *, __int64); 228651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__int64 _InterlockedExchangeAdd64_HLERelease(__int64 volatile *, __int64); 229651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 230f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherchar _InterlockedExchangeAdd8(char volatile *_Addend, char _Value); 231fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 232f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherlong __cdecl _InterlockedIncrement(long volatile *_Addend); 233fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 234f8e56745f5726a8f36444fe7e826e420340544bfEric Christophershort _InterlockedIncrement16(short volatile *_Addend); 235fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 236f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherlong _InterlockedOr(long volatile *_Value, long _Mask); 237fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 238f8e56745f5726a8f36444fe7e826e420340544bfEric Christophershort _InterlockedOr16(short volatile *_Value, short _Mask); 239fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 240f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherchar _InterlockedOr8(char volatile *_Value, char _Mask); 241fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 242f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherlong _InterlockedXor(long volatile *_Value, long _Mask); 243fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 244f8e56745f5726a8f36444fe7e826e420340544bfEric Christophershort _InterlockedXor16(short volatile *_Value, short _Mask); 245fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 246f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherchar _InterlockedXor8(char volatile *_Value, char _Mask); 247f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __cdecl _invpcid(unsigned int, void *); 248fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 249f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned long __cdecl _lrotl(unsigned long, int); 250fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 251f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned long __cdecl _lrotr(unsigned long, int); 252fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 253f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid _ReadBarrier(void); 2540b327d323db4bbd93a4b91ed98ee9ab243290f4eReid Klecknerstatic __inline__ 255f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid _ReadWriteBarrier(void); 256fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 257f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid *_ReturnAddress(void); 258f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned int _rorx_u32(unsigned int, const unsigned int); 259fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 2607f8527bd9284c049b6ff2321952af5057814fa48Eric Christopherunsigned int __cdecl _rotl(unsigned int _Value, int _Shift); 261fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 262f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned short _rotl16(unsigned short _Value, unsigned char _Shift); 263fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 264f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned __int64 __cdecl _rotl64(unsigned __int64 _Value, int _Shift); 265fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 266f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned char _rotl8(unsigned char _Value, unsigned char _Shift); 267fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 268f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned int __cdecl _rotr(unsigned int _Value, int _Shift); 269fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 270f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned short _rotr16(unsigned short _Value, unsigned char _Shift); 271fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 272f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned __int64 __cdecl _rotr64(unsigned __int64 _Value, int _Shift); 273fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 274f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned char _rotr8(unsigned char _Value, unsigned char _Shift); 275f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherint _sarx_i32(int, unsigned int); 276c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines#if __STDC_HOSTED__ 277651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesint __cdecl _setjmp(jmp_buf); 278c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines#endif 279f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned int _shlx_u32(unsigned int, unsigned int); 280f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned int _shrx_u32(unsigned int, unsigned int); 281f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid _Store_HLERelease(long volatile *, long); 282f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid _Store64_HLERelease(__int64 volatile *, __int64); 283f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid _StorePointer_HLERelease(void *volatile *, void *); 2840b327d323db4bbd93a4b91ed98ee9ab243290f4eReid Klecknerstatic __inline__ 285f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid _WriteBarrier(void); 286f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned __int32 xbegin(void); 287f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid _xend(void); 288651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 2890e2c34f92f00628d48968dfea096d36381f494cbStephen Hines#define _XCR_XFEATURE_ENABLED_MASK 0 290f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned __int64 __cdecl _xgetbv(unsigned int); 291f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __cdecl _xsetbv(unsigned int, unsigned __int64); 292f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher 293f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher/* These additional intrinsics are turned on in x64/amd64/x86_64 mode. */ 29469220e76ec232e5659b42f8885c7cec5ee8f1106Warren Hunt#ifdef __x86_64__ 295f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __addgsbyte(unsigned long, unsigned char); 296f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __addgsdword(unsigned long, unsigned long); 297f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __addgsqword(unsigned long, unsigned __int64); 298f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __addgsword(unsigned long, unsigned short); 299651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 300f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __faststorefence(void); 301f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __incgsbyte(unsigned long); 302f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __incgsdword(unsigned long); 303f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __incgsqword(unsigned long); 304f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __incgsword(unsigned long); 305651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesunsigned char __lwpins64(unsigned __int64, unsigned int, unsigned int); 306651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesvoid __lwpval64(unsigned __int64, unsigned int, unsigned int); 307651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesunsigned __int64 __lzcnt64(unsigned __int64); 308651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 309651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesvoid __movsq(unsigned long long *, unsigned long long const *, size_t); 310651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__int64 __mulh(__int64, __int64); 311651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 312f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned __int64 __popcnt64(unsigned __int64); 313651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 314651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesunsigned char __readgsbyte(unsigned long); 315651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 316651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesunsigned long __readgsdword(unsigned long); 317651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 318651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesunsigned __int64 __readgsqword(unsigned long); 319651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesunsigned short __readgsword(unsigned long); 320f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned __int64 __shiftleft128(unsigned __int64 _LowPart, 321f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher unsigned __int64 _HighPart, 322f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher unsigned char _Shift); 323f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned __int64 __shiftright128(unsigned __int64 _LowPart, 324f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher unsigned __int64 _HighPart, 325f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher unsigned char _Shift); 326651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 327f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid __stosq(unsigned __int64 *, unsigned __int64, size_t); 328651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesunsigned char __vmx_on(unsigned __int64 *); 329651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesunsigned char __vmx_vmclear(unsigned __int64 *); 330651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesunsigned char __vmx_vmlaunch(void); 331651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesunsigned char __vmx_vmptrld(unsigned __int64 *); 332651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesunsigned char __vmx_vmread(size_t, size_t *); 333651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesunsigned char __vmx_vmresume(void); 334651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesunsigned char __vmx_vmwrite(size_t, size_t); 335651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesvoid __writegsbyte(unsigned long, unsigned char); 336651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesvoid __writegsdword(unsigned long, unsigned long); 337651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesvoid __writegsqword(unsigned long, unsigned __int64); 338651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesvoid __writegsword(unsigned long, unsigned short); 339fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 340f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask); 341fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 342f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask); 343fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 344f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned char _bittest64(__int64 const *, __int64); 345fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 346f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned char _bittestandcomplement64(__int64 *, __int64); 347fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 348f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned char _bittestandreset64(__int64 *, __int64); 349fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 350f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned char _bittestandset64(__int64 *, __int64); 351f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned __int64 __cdecl _byteswap_uint64(unsigned __int64); 352f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherlong _InterlockedAnd_np(long volatile *_Value, long _Mask); 353f8e56745f5726a8f36444fe7e826e420340544bfEric Christophershort _InterlockedAnd16_np(short volatile *_Value, short _Mask); 354f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher__int64 _InterlockedAnd64_np(__int64 volatile *_Value, __int64 _Mask); 355f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherchar _InterlockedAnd8_np(char volatile *_Value, char _Mask); 356f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned char _interlockedbittestandreset64(__int64 volatile *, __int64); 357651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 358f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned char _interlockedbittestandset64(__int64 volatile *, __int64); 359f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherlong _InterlockedCompareExchange_np(long volatile *_Destination, long _Exchange, 360f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher long _Comparand); 361f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned char _InterlockedCompareExchange128(__int64 volatile *_Destination, 362f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher __int64 _ExchangeHigh, 363f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher __int64 _ExchangeLow, 364f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher __int64 *_CompareandResult); 365f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned char _InterlockedCompareExchange128_np(__int64 volatile *_Destination, 366f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher __int64 _ExchangeHigh, 367f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher __int64 _ExchangeLow, 368f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher __int64 *_ComparandResult); 369f8e56745f5726a8f36444fe7e826e420340544bfEric Christophershort _InterlockedCompareExchange16_np(short volatile *_Destination, 370f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher short _Exchange, short _Comparand); 371651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__int64 _InterlockedCompareExchange64_HLEAcquire(__int64 volatile *, __int64, 372651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines __int64); 373651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__int64 _InterlockedCompareExchange64_HLERelease(__int64 volatile *, __int64, 374651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines __int64); 375f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher__int64 _InterlockedCompareExchange64_np(__int64 volatile *_Destination, 376f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher __int64 _Exchange, __int64 _Comparand); 377651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesvoid *_InterlockedCompareExchangePointer(void *volatile *_Destination, 378651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines void *_Exchange, void *_Comparand); 379f8e56745f5726a8f36444fe7e826e420340544bfEric Christophervoid *_InterlockedCompareExchangePointer_np(void *volatile *_Destination, 380f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher void *_Exchange, void *_Comparand); 381651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 382651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__int64 _InterlockedDecrement64(__int64 volatile *_Addend); 383651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 384651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__int64 _InterlockedExchange64(__int64 volatile *_Target, __int64 _Value); 385651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 386651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__int64 _InterlockedExchangeAdd64(__int64 volatile *_Addend, __int64 _Value); 387651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesvoid *_InterlockedExchangePointer(void *volatile *_Target, void *_Value); 388651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 389651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__int64 _InterlockedIncrement64(__int64 volatile *_Addend); 390f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherlong _InterlockedOr_np(long volatile *_Value, long _Mask); 391f8e56745f5726a8f36444fe7e826e420340544bfEric Christophershort _InterlockedOr16_np(short volatile *_Value, short _Mask); 392651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 393651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__int64 _InterlockedOr64(__int64 volatile *_Value, __int64 _Mask); 39497b94cb783237b3b14e988dced3d2bb48e016edcWarren Hunt__int64 _InterlockedOr64_np(__int64 volatile *_Value, __int64 _Mask); 395f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherchar _InterlockedOr8_np(char volatile *_Value, char _Mask); 396f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherlong _InterlockedXor_np(long volatile *_Value, long _Mask); 397f8e56745f5726a8f36444fe7e826e420340544bfEric Christophershort _InterlockedXor16_np(short volatile *_Value, short _Mask); 398651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 399651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__int64 _InterlockedXor64(__int64 volatile *_Value, __int64 _Mask); 400f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher__int64 _InterlockedXor64_np(__int64 volatile *_Value, __int64 _Mask); 401f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherchar _InterlockedXor8_np(char volatile *_Value, char _Mask); 402651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic __inline__ 403f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher__int64 _mul128(__int64 _Multiplier, __int64 _Multiplicand, 404f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher __int64 *_HighProduct); 405f8e56745f5726a8f36444fe7e826e420340544bfEric Christopherunsigned __int64 _rorx_u64(unsigned __int64, const unsigned int); 406651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__int64 _sarx_i64(__int64, unsigned int); 407c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines#if __STDC_HOSTED__ 408c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hinesint __cdecl _setjmpex(jmp_buf); 409c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines#endif 410651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesunsigned __int64 _shlx_u64(unsigned __int64, unsigned int); 4110e2c34f92f00628d48968dfea096d36381f494cbStephen Hinesunsigned __int64 _shrx_u64(unsigned __int64, unsigned int); 4120e2c34f92f00628d48968dfea096d36381f494cbStephen Hines/* 4130e2c34f92f00628d48968dfea096d36381f494cbStephen Hines * Multiply two 64-bit integers and obtain a 64-bit result. 4140e2c34f92f00628d48968dfea096d36381f494cbStephen Hines * The low-half is returned directly and the high half is in an out parameter. 4150e2c34f92f00628d48968dfea096d36381f494cbStephen Hines */ 41687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned __int64 __DEFAULT_FN_ATTRS 4170e2c34f92f00628d48968dfea096d36381f494cbStephen Hines_umul128(unsigned __int64 _Multiplier, unsigned __int64 _Multiplicand, 4180e2c34f92f00628d48968dfea096d36381f494cbStephen Hines unsigned __int64 *_HighProduct) { 4190e2c34f92f00628d48968dfea096d36381f494cbStephen Hines unsigned __int128 _FullProduct = 4200e2c34f92f00628d48968dfea096d36381f494cbStephen Hines (unsigned __int128)_Multiplier * (unsigned __int128)_Multiplicand; 4210e2c34f92f00628d48968dfea096d36381f494cbStephen Hines *_HighProduct = _FullProduct >> 64; 4220e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return _FullProduct; 4230e2c34f92f00628d48968dfea096d36381f494cbStephen Hines} 42487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned __int64 __DEFAULT_FN_ATTRS 4250e2c34f92f00628d48968dfea096d36381f494cbStephen Hines__umulh(unsigned __int64 _Multiplier, unsigned __int64 _Multiplicand) { 4260e2c34f92f00628d48968dfea096d36381f494cbStephen Hines unsigned __int128 _FullProduct = 4270e2c34f92f00628d48968dfea096d36381f494cbStephen Hines (unsigned __int128)_Multiplier * (unsigned __int128)_Multiplicand; 4280e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return _FullProduct >> 64; 4290e2c34f92f00628d48968dfea096d36381f494cbStephen Hines} 4301fe5ccfaebef277418b0ae44c52d7acf8dea0b0aEric Christopher 43169220e76ec232e5659b42f8885c7cec5ee8f1106Warren Hunt#endif /* __x86_64__ */ 43254fe9cd77b5785ab23b21e5d02fc0406df7e4918Reid Kleckner 433fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt/*----------------------------------------------------------------------------*\ 43487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar|* Multiplication 43587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar\*----------------------------------------------------------------------------*/ 43687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ __int64 __DEFAULT_FN_ATTRS 43787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar__emul(int __in1, int __in2) { 43887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return (__int64)__in1 * (__int64)__in2; 43987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 44087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned __int64 __DEFAULT_FN_ATTRS 44187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar__emulu(unsigned int __in1, unsigned int __in2) { 44287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return (unsigned __int64)__in1 * (unsigned __int64)__in2; 44387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 44487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar/*----------------------------------------------------------------------------*\ 445fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt|* Bit Twiddling 446fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt\*----------------------------------------------------------------------------*/ 44787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned char __DEFAULT_FN_ATTRS 448fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_rotl8(unsigned char _Value, unsigned char _Shift) { 449fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt _Shift &= 0x7; 450fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt return _Shift ? (_Value << _Shift) | (_Value >> (8 - _Shift)) : _Value; 451fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 45287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned char __DEFAULT_FN_ATTRS 453fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_rotr8(unsigned char _Value, unsigned char _Shift) { 454fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt _Shift &= 0x7; 455fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt return _Shift ? (_Value >> _Shift) | (_Value << (8 - _Shift)) : _Value; 456fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 45787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned short __DEFAULT_FN_ATTRS 458fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_rotl16(unsigned short _Value, unsigned char _Shift) { 459fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt _Shift &= 0xf; 460fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt return _Shift ? (_Value << _Shift) | (_Value >> (16 - _Shift)) : _Value; 461fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 46287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned short __DEFAULT_FN_ATTRS 463fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_rotr16(unsigned short _Value, unsigned char _Shift) { 464fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt _Shift &= 0xf; 465fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt return _Shift ? (_Value >> _Shift) | (_Value << (16 - _Shift)) : _Value; 466fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 46787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned int __DEFAULT_FN_ATTRS 468fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_rotl(unsigned int _Value, int _Shift) { 469fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt _Shift &= 0x1f; 470fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt return _Shift ? (_Value << _Shift) | (_Value >> (32 - _Shift)) : _Value; 471fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 47287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned int __DEFAULT_FN_ATTRS 473fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_rotr(unsigned int _Value, int _Shift) { 474fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt _Shift &= 0x1f; 475fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt return _Shift ? (_Value >> _Shift) | (_Value << (32 - _Shift)) : _Value; 476fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 47787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned long __DEFAULT_FN_ATTRS 478fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_lrotl(unsigned long _Value, int _Shift) { 479fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt _Shift &= 0x1f; 480fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt return _Shift ? (_Value << _Shift) | (_Value >> (32 - _Shift)) : _Value; 481fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 48287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned long __DEFAULT_FN_ATTRS 483fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_lrotr(unsigned long _Value, int _Shift) { 484fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt _Shift &= 0x1f; 485fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt return _Shift ? (_Value >> _Shift) | (_Value << (32 - _Shift)) : _Value; 486fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 487fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic 48887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar__inline__ unsigned __int64 __DEFAULT_FN_ATTRS 489fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_rotl64(unsigned __int64 _Value, int _Shift) { 490fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt _Shift &= 0x3f; 491fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt return _Shift ? (_Value << _Shift) | (_Value >> (64 - _Shift)) : _Value; 492fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 493fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic 49487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar__inline__ unsigned __int64 __DEFAULT_FN_ATTRS 495fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_rotr64(unsigned __int64 _Value, int _Shift) { 496fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt _Shift &= 0x3f; 497fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt return _Shift ? (_Value >> _Shift) | (_Value << (64 - _Shift)) : _Value; 498fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 499fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt/*----------------------------------------------------------------------------*\ 500fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt|* Bit Counting and Testing 501fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt\*----------------------------------------------------------------------------*/ 50287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned char __DEFAULT_FN_ATTRS 503fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_BitScanForward(unsigned long *_Index, unsigned long _Mask) { 504fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt if (!_Mask) 505fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt return 0; 506fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt *_Index = __builtin_ctzl(_Mask); 507fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt return 1; 508fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 50987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned char __DEFAULT_FN_ATTRS 510fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_BitScanReverse(unsigned long *_Index, unsigned long _Mask) { 511fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt if (!_Mask) 512fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt return 0; 513fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt *_Index = 31 - __builtin_clzl(_Mask); 514fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt return 1; 515fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 51687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned short __DEFAULT_FN_ATTRS 51787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar__popcnt16(unsigned short _Value) { 51887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return __builtin_popcount((int)_Value); 51987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 52087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned int __DEFAULT_FN_ATTRS 52187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar__popcnt(unsigned int _Value) { 52287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return __builtin_popcount(_Value); 52387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 52487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned char __DEFAULT_FN_ATTRS 52587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar_bittest(long const *_BitBase, long _BitPos) { 52687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return (*_BitBase >> _BitPos) & 1; 52787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 52887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned char __DEFAULT_FN_ATTRS 52987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar_bittestandcomplement(long *_BitBase, long _BitPos) { 53087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar unsigned char _Res = (*_BitBase >> _BitPos) & 1; 53187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar *_BitBase = *_BitBase ^ (1 << _BitPos); 53287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return _Res; 53387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 53487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned char __DEFAULT_FN_ATTRS 53587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar_bittestandreset(long *_BitBase, long _BitPos) { 53687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar unsigned char _Res = (*_BitBase >> _BitPos) & 1; 53787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar *_BitBase = *_BitBase & ~(1 << _BitPos); 53887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return _Res; 53987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 54087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned char __DEFAULT_FN_ATTRS 54187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar_bittestandset(long *_BitBase, long _BitPos) { 54287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar unsigned char _Res = (*_BitBase >> _BitPos) & 1; 54387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar *_BitBase = *_BitBase | (1 << _BitPos); 54487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return _Res; 54587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 54687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned char __DEFAULT_FN_ATTRS 54787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar_interlockedbittestandset(long volatile *_BitBase, long _BitPos) { 54887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar long _PrevVal = __atomic_fetch_or(_BitBase, 1l << _BitPos, __ATOMIC_SEQ_CST); 54987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return (_PrevVal >> _BitPos) & 1; 550fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 55169220e76ec232e5659b42f8885c7cec5ee8f1106Warren Hunt#ifdef __x86_64__ 55287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned char __DEFAULT_FN_ATTRS 553fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask) { 554fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt if (!_Mask) 555fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt return 0; 556fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt *_Index = __builtin_ctzll(_Mask); 557fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt return 1; 558fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 55987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned char __DEFAULT_FN_ATTRS 560fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask) { 561fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt if (!_Mask) 562fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt return 0; 563fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt *_Index = 63 - __builtin_clzll(_Mask); 564fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt return 1; 565fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 566fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Huntstatic __inline__ 56787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarunsigned __int64 __DEFAULT_FN_ATTRS 56887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar__popcnt64(unsigned __int64 _Value) { 56987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return __builtin_popcountll(_Value); 57087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 57187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned char __DEFAULT_FN_ATTRS 57287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar_bittest64(__int64 const *_BitBase, __int64 _BitPos) { 57387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return (*_BitBase >> _BitPos) & 1; 57487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 57587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned char __DEFAULT_FN_ATTRS 57687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar_bittestandcomplement64(__int64 *_BitBase, __int64 _BitPos) { 57787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar unsigned char _Res = (*_BitBase >> _BitPos) & 1; 57887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar *_BitBase = *_BitBase ^ (1ll << _BitPos); 57987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return _Res; 58087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 58187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned char __DEFAULT_FN_ATTRS 58287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar_bittestandreset64(__int64 *_BitBase, __int64 _BitPos) { 58387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar unsigned char _Res = (*_BitBase >> _BitPos) & 1; 58487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar *_BitBase = *_BitBase & ~(1ll << _BitPos); 58587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return _Res; 58687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 58787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned char __DEFAULT_FN_ATTRS 58887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar_bittestandset64(__int64 *_BitBase, __int64 _BitPos) { 58987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar unsigned char _Res = (*_BitBase >> _BitPos) & 1; 59087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar *_BitBase = *_BitBase | (1ll << _BitPos); 59187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return _Res; 59287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 59387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned char __DEFAULT_FN_ATTRS 59487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar_interlockedbittestandset64(__int64 volatile *_BitBase, __int64 _BitPos) { 59587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar long long _PrevVal = 59687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar __atomic_fetch_or(_BitBase, 1ll << _BitPos, __ATOMIC_SEQ_CST); 59787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return (_PrevVal >> _BitPos) & 1; 598651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 599fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt#endif 600fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt/*----------------------------------------------------------------------------*\ 601fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt|* Interlocked Exchange Add 602fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt\*----------------------------------------------------------------------------*/ 60387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ char __DEFAULT_FN_ATTRS 604fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedExchangeAdd8(char volatile *_Addend, char _Value) { 60587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return __atomic_fetch_add(_Addend, _Value, __ATOMIC_SEQ_CST); 606fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 60787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ short __DEFAULT_FN_ATTRS 608fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedExchangeAdd16(short volatile *_Addend, short _Value) { 60987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return __atomic_fetch_add(_Addend, _Value, __ATOMIC_SEQ_CST); 610fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 61169220e76ec232e5659b42f8885c7cec5ee8f1106Warren Hunt#ifdef __x86_64__ 61287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ __int64 __DEFAULT_FN_ATTRS 613fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedExchangeAdd64(__int64 volatile *_Addend, __int64 _Value) { 61487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return __atomic_fetch_add(_Addend, _Value, __ATOMIC_SEQ_CST); 615fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 616fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt#endif 617fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt/*----------------------------------------------------------------------------*\ 618fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt|* Interlocked Exchange Sub 619fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt\*----------------------------------------------------------------------------*/ 62087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ char __DEFAULT_FN_ATTRS 621fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedExchangeSub8(char volatile *_Subend, char _Value) { 62287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return __atomic_fetch_sub(_Subend, _Value, __ATOMIC_SEQ_CST); 623fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 62487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ short __DEFAULT_FN_ATTRS 625fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedExchangeSub16(short volatile *_Subend, short _Value) { 62687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return __atomic_fetch_sub(_Subend, _Value, __ATOMIC_SEQ_CST); 627fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 62887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ long __DEFAULT_FN_ATTRS 629fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedExchangeSub(long volatile *_Subend, long _Value) { 63087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return __atomic_fetch_sub(_Subend, _Value, __ATOMIC_SEQ_CST); 631fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 63269220e76ec232e5659b42f8885c7cec5ee8f1106Warren Hunt#ifdef __x86_64__ 63387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ __int64 __DEFAULT_FN_ATTRS 634fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedExchangeSub64(__int64 volatile *_Subend, __int64 _Value) { 63587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return __atomic_fetch_sub(_Subend, _Value, __ATOMIC_SEQ_CST); 636fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 637fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt#endif 638fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt/*----------------------------------------------------------------------------*\ 639fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt|* Interlocked Increment 640fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt\*----------------------------------------------------------------------------*/ 64187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ short __DEFAULT_FN_ATTRS 642651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines_InterlockedIncrement16(short volatile *_Value) { 64387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return __atomic_add_fetch(_Value, 1, __ATOMIC_SEQ_CST); 644fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 64569220e76ec232e5659b42f8885c7cec5ee8f1106Warren Hunt#ifdef __x86_64__ 64687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ __int64 __DEFAULT_FN_ATTRS 647fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedIncrement64(__int64 volatile *_Value) { 64887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return __atomic_add_fetch(_Value, 1, __ATOMIC_SEQ_CST); 649fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 650fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt#endif 651fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt/*----------------------------------------------------------------------------*\ 652fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt|* Interlocked Decrement 653fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt\*----------------------------------------------------------------------------*/ 65487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ short __DEFAULT_FN_ATTRS 655651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines_InterlockedDecrement16(short volatile *_Value) { 65687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return __atomic_sub_fetch(_Value, 1, __ATOMIC_SEQ_CST); 657fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 65869220e76ec232e5659b42f8885c7cec5ee8f1106Warren Hunt#ifdef __x86_64__ 65987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ __int64 __DEFAULT_FN_ATTRS 660fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedDecrement64(__int64 volatile *_Value) { 66187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return __atomic_sub_fetch(_Value, 1, __ATOMIC_SEQ_CST); 662fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 663fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt#endif 664fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt/*----------------------------------------------------------------------------*\ 665fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt|* Interlocked And 666fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt\*----------------------------------------------------------------------------*/ 66787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ char __DEFAULT_FN_ATTRS 668fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedAnd8(char volatile *_Value, char _Mask) { 6694967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return __atomic_fetch_and(_Value, _Mask, __ATOMIC_SEQ_CST); 670fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 67187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ short __DEFAULT_FN_ATTRS 672fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedAnd16(short volatile *_Value, short _Mask) { 6734967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return __atomic_fetch_and(_Value, _Mask, __ATOMIC_SEQ_CST); 674fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 67587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ long __DEFAULT_FN_ATTRS 676fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedAnd(long volatile *_Value, long _Mask) { 6774967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return __atomic_fetch_and(_Value, _Mask, __ATOMIC_SEQ_CST); 678fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 67969220e76ec232e5659b42f8885c7cec5ee8f1106Warren Hunt#ifdef __x86_64__ 68087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ __int64 __DEFAULT_FN_ATTRS 681fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedAnd64(__int64 volatile *_Value, __int64 _Mask) { 6824967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return __atomic_fetch_and(_Value, _Mask, __ATOMIC_SEQ_CST); 683fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 684fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt#endif 685fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt/*----------------------------------------------------------------------------*\ 686fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt|* Interlocked Or 687fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt\*----------------------------------------------------------------------------*/ 68887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ char __DEFAULT_FN_ATTRS 689fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedOr8(char volatile *_Value, char _Mask) { 6904967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return __atomic_fetch_or(_Value, _Mask, __ATOMIC_SEQ_CST); 691fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 69287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ short __DEFAULT_FN_ATTRS 693fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedOr16(short volatile *_Value, short _Mask) { 6944967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return __atomic_fetch_or(_Value, _Mask, __ATOMIC_SEQ_CST); 695fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 69687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ long __DEFAULT_FN_ATTRS 697fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedOr(long volatile *_Value, long _Mask) { 6984967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return __atomic_fetch_or(_Value, _Mask, __ATOMIC_SEQ_CST); 699fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 70069220e76ec232e5659b42f8885c7cec5ee8f1106Warren Hunt#ifdef __x86_64__ 70187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ __int64 __DEFAULT_FN_ATTRS 702fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedOr64(__int64 volatile *_Value, __int64 _Mask) { 7034967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return __atomic_fetch_or(_Value, _Mask, __ATOMIC_SEQ_CST); 704fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 705fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt#endif 706fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt/*----------------------------------------------------------------------------*\ 707fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt|* Interlocked Xor 708fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt\*----------------------------------------------------------------------------*/ 70987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ char __DEFAULT_FN_ATTRS 710fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedXor8(char volatile *_Value, char _Mask) { 7114967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_SEQ_CST); 712fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 71387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ short __DEFAULT_FN_ATTRS 714fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedXor16(short volatile *_Value, short _Mask) { 7154967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_SEQ_CST); 716fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 71787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ long __DEFAULT_FN_ATTRS 718fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedXor(long volatile *_Value, long _Mask) { 7194967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_SEQ_CST); 720fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 72169220e76ec232e5659b42f8885c7cec5ee8f1106Warren Hunt#ifdef __x86_64__ 72287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ __int64 __DEFAULT_FN_ATTRS 723fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedXor64(__int64 volatile *_Value, __int64 _Mask) { 7244967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_SEQ_CST); 725fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 726fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt#endif 727fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt/*----------------------------------------------------------------------------*\ 728fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt|* Interlocked Exchange 729fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt\*----------------------------------------------------------------------------*/ 73087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ char __DEFAULT_FN_ATTRS 731fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedExchange8(char volatile *_Target, char _Value) { 73287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_SEQ_CST); 733fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt return _Value; 734fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 73587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ short __DEFAULT_FN_ATTRS 736fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedExchange16(short volatile *_Target, short _Value) { 73787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_SEQ_CST); 738fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt return _Value; 739fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 74069220e76ec232e5659b42f8885c7cec5ee8f1106Warren Hunt#ifdef __x86_64__ 74187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ __int64 __DEFAULT_FN_ATTRS 742fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedExchange64(__int64 volatile *_Target, __int64 _Value) { 74387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_SEQ_CST); 744fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt return _Value; 745fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 746fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt#endif 747fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt/*----------------------------------------------------------------------------*\ 748fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt|* Interlocked Compare Exchange 749fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt\*----------------------------------------------------------------------------*/ 75087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ char __DEFAULT_FN_ATTRS 751fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedCompareExchange8(char volatile *_Destination, 752fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt char _Exchange, char _Comparand) { 75387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, 75487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); 755fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt return _Comparand; 756fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 75787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ short __DEFAULT_FN_ATTRS 758fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedCompareExchange16(short volatile *_Destination, 759fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt short _Exchange, short _Comparand) { 76087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, 76187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); 762fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt return _Comparand; 763fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 76487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ __int64 __DEFAULT_FN_ATTRS 765fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_InterlockedCompareExchange64(__int64 volatile *_Destination, 766fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt __int64 _Exchange, __int64 _Comparand) { 76787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, 76887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); 769fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt return _Comparand; 770fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 771fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt/*----------------------------------------------------------------------------*\ 7720b327d323db4bbd93a4b91ed98ee9ab243290f4eReid Kleckner|* Barriers 7730b327d323db4bbd93a4b91ed98ee9ab243290f4eReid Kleckner\*----------------------------------------------------------------------------*/ 77487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ void __DEFAULT_FN_ATTRS 7750e2c34f92f00628d48968dfea096d36381f494cbStephen Hines__attribute__((__deprecated__("use other intrinsics or C++11 atomics instead"))) 7760b327d323db4bbd93a4b91ed98ee9ab243290f4eReid Kleckner_ReadWriteBarrier(void) { 77787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar __atomic_signal_fence(__ATOMIC_SEQ_CST); 7780b327d323db4bbd93a4b91ed98ee9ab243290f4eReid Kleckner} 77987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ void __DEFAULT_FN_ATTRS 7800e2c34f92f00628d48968dfea096d36381f494cbStephen Hines__attribute__((__deprecated__("use other intrinsics or C++11 atomics instead"))) 7810b327d323db4bbd93a4b91ed98ee9ab243290f4eReid Kleckner_ReadBarrier(void) { 78287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar __atomic_signal_fence(__ATOMIC_SEQ_CST); 7830b327d323db4bbd93a4b91ed98ee9ab243290f4eReid Kleckner} 78487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ void __DEFAULT_FN_ATTRS 7850e2c34f92f00628d48968dfea096d36381f494cbStephen Hines__attribute__((__deprecated__("use other intrinsics or C++11 atomics instead"))) 7860b327d323db4bbd93a4b91ed98ee9ab243290f4eReid Kleckner_WriteBarrier(void) { 78787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar __atomic_signal_fence(__ATOMIC_SEQ_CST); 7880b327d323db4bbd93a4b91ed98ee9ab243290f4eReid Kleckner} 789651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#ifdef __x86_64__ 79087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ void __DEFAULT_FN_ATTRS 791651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__faststorefence(void) { 79287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar __atomic_thread_fence(__ATOMIC_SEQ_CST); 793651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 794651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#endif 795651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines/*----------------------------------------------------------------------------*\ 796651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines|* readfs, readgs 797651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines|* (Pointers in address space #256 and #257 are relative to the GS and FS 798651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines|* segment registers, respectively.) 799651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines\*----------------------------------------------------------------------------*/ 800651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#define __ptr_to_addr_space(__addr_space_nbr, __type, __offset) \ 801651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ((volatile __type __attribute__((__address_space__(__addr_space_nbr)))*) \ 802651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines (__offset)) 803651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 804651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#ifdef __i386__ 80587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned char __DEFAULT_FN_ATTRS 806651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__readfsbyte(unsigned long __offset) { 807651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return *__ptr_to_addr_space(257, unsigned char, __offset); 808651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 80987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned short __DEFAULT_FN_ATTRS 810651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__readfsword(unsigned long __offset) { 811651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return *__ptr_to_addr_space(257, unsigned short, __offset); 812651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 8134967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainarstatic __inline__ unsigned __int64 __DEFAULT_FN_ATTRS 8144967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar__readfsqword(unsigned long __offset) { 8154967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return *__ptr_to_addr_space(257, unsigned __int64, __offset); 8164967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar} 817651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#endif 818651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#ifdef __x86_64__ 81987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned char __DEFAULT_FN_ATTRS 820651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__readgsbyte(unsigned long __offset) { 821651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return *__ptr_to_addr_space(256, unsigned char, __offset); 822651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 8234967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainarstatic __inline__ unsigned short __DEFAULT_FN_ATTRS 8244967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar__readgsword(unsigned long __offset) { 8254967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return *__ptr_to_addr_space(256, unsigned short, __offset); 8264967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar} 82787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned long __DEFAULT_FN_ATTRS 828651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__readgsdword(unsigned long __offset) { 829651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return *__ptr_to_addr_space(256, unsigned long, __offset); 830651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 83187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned __int64 __DEFAULT_FN_ATTRS 832651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__readgsqword(unsigned long __offset) { 833651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return *__ptr_to_addr_space(256, unsigned __int64, __offset); 834651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 835651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#endif 836651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#undef __ptr_to_addr_space 837651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines/*----------------------------------------------------------------------------*\ 838651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines|* movs, stos 839651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines\*----------------------------------------------------------------------------*/ 840c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines#if defined(__i386__) || defined(__x86_64__) 84187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ void __DEFAULT_FN_ATTRS 842651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__movsb(unsigned char *__dst, unsigned char const *__src, size_t __n) { 843651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines __asm__("rep movsb" : : "D"(__dst), "S"(__src), "c"(__n) 844651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : "%edi", "%esi", "%ecx"); 845651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 84687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ void __DEFAULT_FN_ATTRS 847651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__movsd(unsigned long *__dst, unsigned long const *__src, size_t __n) { 848651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines __asm__("rep movsl" : : "D"(__dst), "S"(__src), "c"(__n) 849651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : "%edi", "%esi", "%ecx"); 850651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 85187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ void __DEFAULT_FN_ATTRS 852651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__movsw(unsigned short *__dst, unsigned short const *__src, size_t __n) { 85387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar __asm__("rep movsw" : : "D"(__dst), "S"(__src), "c"(__n) 854651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : "%edi", "%esi", "%ecx"); 855651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 85687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ void __DEFAULT_FN_ATTRS 857651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__stosb(unsigned char *__dst, unsigned char __x, size_t __n) { 858651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines __asm__("rep stosb" : : "D"(__dst), "a"(__x), "c"(__n) 859651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : "%edi", "%ecx"); 860651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 86187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ void __DEFAULT_FN_ATTRS 862651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__stosd(unsigned long *__dst, unsigned long __x, size_t __n) { 863651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines __asm__("rep stosl" : : "D"(__dst), "a"(__x), "c"(__n) 864651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : "%edi", "%ecx"); 865651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 86687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ void __DEFAULT_FN_ATTRS 867651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__stosw(unsigned short *__dst, unsigned short __x, size_t __n) { 86887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar __asm__("rep stosw" : : "D"(__dst), "a"(__x), "c"(__n) 869651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : "%edi", "%ecx"); 870651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 871c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines#endif 872651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#ifdef __x86_64__ 87387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ void __DEFAULT_FN_ATTRS 874651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__movsq(unsigned long long *__dst, unsigned long long const *__src, size_t __n) { 875651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines __asm__("rep movsq" : : "D"(__dst), "S"(__src), "c"(__n) 876651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : "%edi", "%esi", "%ecx"); 877651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 87887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ void __DEFAULT_FN_ATTRS 879651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__stosq(unsigned __int64 *__dst, unsigned __int64 __x, size_t __n) { 880651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines __asm__("rep stosq" : : "D"(__dst), "a"(__x), "c"(__n) 881651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : "%edi", "%ecx"); 882651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 883651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#endif 884651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 8850b327d323db4bbd93a4b91ed98ee9ab243290f4eReid Kleckner/*----------------------------------------------------------------------------*\ 886fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt|* Misc 887fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt\*----------------------------------------------------------------------------*/ 88887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ void * __DEFAULT_FN_ATTRS 889fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_AddressOfReturnAddress(void) { 890fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt return (void*)((char*)__builtin_frame_address(0) + sizeof(void*)); 891fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 89287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ void * __DEFAULT_FN_ATTRS 893fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt_ReturnAddress(void) { 894fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt return __builtin_return_address(0); 895fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt} 896c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines#if defined(__i386__) || defined(__x86_64__) 89787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ void __DEFAULT_FN_ATTRS 898651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__cpuid(int __info[4], int __level) { 899651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines __asm__ ("cpuid" : "=a"(__info[0]), "=b" (__info[1]), "=c"(__info[2]), "=d"(__info[3]) 900651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : "a"(__level)); 901651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 90287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ void __DEFAULT_FN_ATTRS 903651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__cpuidex(int __info[4], int __level, int __ecx) { 904651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines __asm__ ("cpuid" : "=a"(__info[0]), "=b" (__info[1]), "=c"(__info[2]), "=d"(__info[3]) 905651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : "a"(__level), "c"(__ecx)); 906651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 90787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned __int64 __cdecl __DEFAULT_FN_ATTRS 908651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines_xgetbv(unsigned int __xcr_no) { 909651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned int __eax, __edx; 910651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines __asm__ ("xgetbv" : "=a" (__eax), "=d" (__edx) : "c" (__xcr_no)); 911651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return ((unsigned __int64)__edx << 32) | __eax; 912651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 91387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ void __DEFAULT_FN_ATTRS 914651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines__halt(void) { 915651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines __asm__ volatile ("hlt"); 916651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 917c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines#endif 918fffce2e7410e4b52b887e7a0d43b0af97f2c18e8Warren Hunt 9196bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines/*----------------------------------------------------------------------------*\ 9206bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines|* Privileged intrinsics 9216bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines\*----------------------------------------------------------------------------*/ 922c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines#if defined(__i386__) || defined(__x86_64__) 92387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned __int64 __DEFAULT_FN_ATTRS 9246bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines__readmsr(unsigned long __register) { 9256bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // Loads the contents of a 64-bit model specific register (MSR) specified in 9266bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // the ECX register into registers EDX:EAX. The EDX register is loaded with 9276bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // the high-order 32 bits of the MSR and the EAX register is loaded with the 9286bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // low-order 32 bits. If less than 64 bits are implemented in the MSR being 9296bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // read, the values returned to EDX:EAX in unimplemented bit locations are 9306bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // undefined. 9316bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines unsigned long __edx; 9326bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines unsigned long __eax; 9336bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines __asm__ ("rdmsr" : "=d"(__edx), "=a"(__eax) : "c"(__register)); 9346bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return (((unsigned __int64)__edx) << 32) | (unsigned __int64)__eax; 9356bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines} 9366bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 93787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ unsigned long __DEFAULT_FN_ATTRS 9386bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines__readcr3(void) { 9396bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines unsigned long __cr3_val; 9406bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines __asm__ __volatile__ ("mov %%cr3, %0" : "=q"(__cr3_val) : : "memory"); 9416bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return __cr3_val; 9426bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines} 9436bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 94487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic __inline__ void __DEFAULT_FN_ATTRS 9456bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines__writecr3(unsigned int __cr3_val) { 9466bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines __asm__ ("mov %0, %%cr3" : : "q"(__cr3_val) : "memory"); 9476bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines} 948c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines#endif 9496bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 950b00ddc5b5f66312e46fcfc95aad953a0abb66b12Eric Christopher#ifdef __cplusplus 9511fe5ccfaebef277418b0ae44c52d7acf8dea0b0aEric Christopher} 952f8e56745f5726a8f36444fe7e826e420340544bfEric Christopher#endif 95354fe9cd77b5785ab23b21e5d02fc0406df7e4918Reid Kleckner 95487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar#undef __DEFAULT_FN_ATTRS 95587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 95654fe9cd77b5785ab23b21e5d02fc0406df7e4918Reid Kleckner#endif /* __INTRIN_H */ 95754fe9cd77b5785ab23b21e5d02fc0406df7e4918Reid Kleckner#endif /* _MSC_VER */ 958