Intrin.h revision 651f13cea278ec967336033dd032faef0e9fc2ec
1/* ===-------- Intrin.h ---------------------------------------------------=== 2 * 3 * Permission is hereby granted, free of charge, to any person obtaining a copy 4 * of this software and associated documentation files (the "Software"), to deal 5 * in the Software without restriction, including without limitation the rights 6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 * copies of the Software, and to permit persons to whom the Software is 8 * furnished to do so, subject to the following conditions: 9 * 10 * The above copyright notice and this permission notice shall be included in 11 * all copies or substantial portions of the Software. 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 * THE SOFTWARE. 20 * 21 *===-----------------------------------------------------------------------=== 22 */ 23 24/* Only include this if we're compiling for the windows platform. */ 25#ifndef _MSC_VER 26#include_next <Intrin.h> 27#else 28 29#ifndef __INTRIN_H 30#define __INTRIN_H 31 32/* First include the standard intrinsics. */ 33#include <x86intrin.h> 34 35/* For the definition of jmp_buf. */ 36#include <setjmp.h> 37 38#ifdef __cplusplus 39extern "C" { 40#endif 41 42/* And the random ones that aren't in those files. */ 43__m64 _m_from_float(float); 44__m64 _m_from_int(int _l); 45void _m_prefetch(void *); 46float _m_to_float(__m64); 47int _m_to_int(__m64 _M); 48 49/* Other assorted instruction intrinsics. */ 50void __addfsbyte(unsigned long, unsigned char); 51void __addfsdword(unsigned long, unsigned long); 52void __addfsword(unsigned long, unsigned short); 53void __code_seg(const char *); 54static __inline__ 55void __cpuid(int[4], int); 56static __inline__ 57void __cpuidex(int[4], int, int); 58void __debugbreak(void); 59__int64 __emul(int, int); 60unsigned __int64 __emulu(unsigned int, unsigned int); 61void __cdecl __fastfail(unsigned int); 62unsigned int __getcallerseflags(void); 63static __inline__ 64void __halt(void); 65unsigned char __inbyte(unsigned short); 66void __inbytestring(unsigned short, unsigned char *, unsigned long); 67void __incfsbyte(unsigned long); 68void __incfsdword(unsigned long); 69void __incfsword(unsigned long); 70unsigned long __indword(unsigned short); 71void __indwordstring(unsigned short, unsigned long *, unsigned long); 72void __int2c(void); 73void __invlpg(void *); 74unsigned short __inword(unsigned short); 75void __inwordstring(unsigned short, unsigned short *, unsigned long); 76void __lidt(void *); 77unsigned __int64 __ll_lshift(unsigned __int64, int); 78__int64 __ll_rshift(__int64, int); 79void __llwpcb(void *); 80unsigned char __lwpins32(unsigned int, unsigned int, unsigned int); 81void __lwpval32(unsigned int, unsigned int, unsigned int); 82unsigned int __lzcnt(unsigned int); 83unsigned short __lzcnt16(unsigned short); 84static __inline__ 85void __movsb(unsigned char *, unsigned char const *, size_t); 86static __inline__ 87void __movsd(unsigned long *, unsigned long const *, size_t); 88static __inline__ 89void __movsw(unsigned short *, unsigned short const *, size_t); 90void __nop(void); 91void __nvreg_restore_fence(void); 92void __nvreg_save_fence(void); 93void __outbyte(unsigned short, unsigned char); 94void __outbytestring(unsigned short, unsigned char *, unsigned long); 95void __outdword(unsigned short, unsigned long); 96void __outdwordstring(unsigned short, unsigned long *, unsigned long); 97void __outword(unsigned short, unsigned short); 98void __outwordstring(unsigned short, unsigned short *, unsigned long); 99static __inline__ 100unsigned int __popcnt(unsigned int); 101static __inline__ 102unsigned short __popcnt16(unsigned short); 103static __inline__ 104unsigned __int64 __rdtsc(void); 105unsigned __int64 __rdtscp(unsigned int *); 106unsigned long __readcr0(void); 107unsigned long __readcr2(void); 108unsigned long __readcr3(void); 109unsigned long __readcr4(void); 110unsigned long __readcr8(void); 111unsigned int __readdr(unsigned int); 112#ifdef __i386__ 113static __inline__ 114unsigned char __readfsbyte(unsigned long); 115static __inline__ 116unsigned long __readfsdword(unsigned long); 117static __inline__ 118unsigned __int64 __readfsqword(unsigned long); 119static __inline__ 120unsigned short __readfsword(unsigned long); 121#endif 122unsigned __int64 __readmsr(unsigned long); 123unsigned __int64 __readpmc(unsigned long); 124unsigned long __segmentlimit(unsigned long); 125void __sidt(void *); 126void *__slwpcb(void); 127static __inline__ 128void __stosb(unsigned char *, unsigned char, size_t); 129static __inline__ 130void __stosd(unsigned long *, unsigned long, size_t); 131static __inline__ 132void __stosw(unsigned short *, unsigned short, size_t); 133void __svm_clgi(void); 134void __svm_invlpga(void *, int); 135void __svm_skinit(int); 136void __svm_stgi(void); 137void __svm_vmload(size_t); 138void __svm_vmrun(size_t); 139void __svm_vmsave(size_t); 140void __ud2(void); 141unsigned __int64 __ull_rshift(unsigned __int64, int); 142void __vmx_off(void); 143void __vmx_vmptrst(unsigned __int64 *); 144void __wbinvd(void); 145void __writecr0(unsigned int); 146void __writecr3(unsigned int); 147void __writecr4(unsigned int); 148void __writecr8(unsigned int); 149void __writedr(unsigned int, unsigned int); 150void __writefsbyte(unsigned long, unsigned char); 151void __writefsdword(unsigned long, unsigned long); 152void __writefsqword(unsigned long, unsigned __int64); 153void __writefsword(unsigned long, unsigned short); 154void __writemsr(unsigned long, unsigned __int64); 155static __inline__ 156void *_AddressOfReturnAddress(void); 157unsigned int _andn_u32(unsigned int, unsigned int); 158unsigned int _bextr_u32(unsigned int, unsigned int, unsigned int); 159unsigned int _bextri_u32(unsigned int, unsigned int); 160static __inline__ 161unsigned char _BitScanForward(unsigned long *_Index, unsigned long _Mask); 162static __inline__ 163unsigned char _BitScanReverse(unsigned long *_Index, unsigned long _Mask); 164static __inline__ 165unsigned char _bittest(long const *, long); 166static __inline__ 167unsigned char _bittestandcomplement(long *, long); 168static __inline__ 169unsigned char _bittestandreset(long *, long); 170static __inline__ 171unsigned char _bittestandset(long *, long); 172unsigned int _blcfill_u32(unsigned int); 173unsigned int _blci_u32(unsigned int); 174unsigned int _blcic_u32(unsigned int); 175unsigned int _blcmsk_u32(unsigned int); 176unsigned int _blcs_u32(unsigned int); 177unsigned int _blsfill_u32(unsigned int); 178unsigned int _blsi_u32(unsigned int); 179unsigned int _blsic_u32(unsigned int); 180unsigned int _blsmsk_u32(unsigned int); 181unsigned int _blsr_u32(unsigned int); 182unsigned __int64 __cdecl _byteswap_uint64(unsigned __int64); 183unsigned long __cdecl _byteswap_ulong(unsigned long); 184unsigned short __cdecl _byteswap_ushort(unsigned short); 185unsigned _bzhi_u32(unsigned int, unsigned int); 186void __cdecl _disable(void); 187void __cdecl _enable(void); 188void __cdecl _fxrstor(void const *); 189void __cdecl _fxsave(void *); 190long _InterlockedAddLargeStatistic(__int64 volatile *_Addend, long _Value); 191static __inline__ 192long _InterlockedAnd(long volatile *_Value, long _Mask); 193static __inline__ 194short _InterlockedAnd16(short volatile *_Value, short _Mask); 195static __inline__ 196char _InterlockedAnd8(char volatile *_Value, char _Mask); 197unsigned char _interlockedbittestandreset(long volatile *, long); 198static __inline__ 199unsigned char _interlockedbittestandset(long volatile *, long); 200static __inline__ 201long __cdecl _InterlockedCompareExchange(long volatile *_Destination, 202 long _Exchange, long _Comparand); 203long _InterlockedCompareExchange_HLEAcquire(long volatile *, long, long); 204long _InterlockedCompareExchange_HLERelease(long volatile *, long, long); 205static __inline__ 206short _InterlockedCompareExchange16(short volatile *_Destination, 207 short _Exchange, short _Comparand); 208static __inline__ 209__int64 _InterlockedCompareExchange64(__int64 volatile *_Destination, 210 __int64 _Exchange, __int64 _Comparand); 211__int64 _InterlockedcompareExchange64_HLEAcquire(__int64 volatile *, __int64, 212 __int64); 213__int64 _InterlockedCompareExchange64_HLERelease(__int64 volatile *, __int64, 214 __int64); 215static __inline__ 216char _InterlockedCompareExchange8(char volatile *_Destination, char _Exchange, 217 char _Comparand); 218void *_InterlockedCompareExchangePointer_HLEAcquire(void *volatile *, void *, 219 void *); 220void *_InterlockedCompareExchangePointer_HLERelease(void *volatile *, void *, 221 void *); 222static __inline__ 223long __cdecl _InterlockedDecrement(long volatile *_Addend); 224static __inline__ 225short _InterlockedDecrement16(short volatile *_Addend); 226static __inline__ 227long __cdecl _InterlockedExchange(long volatile *_Target, long _Value); 228static __inline__ 229short _InterlockedExchange16(short volatile *_Target, short _Value); 230static __inline__ 231char _InterlockedExchange8(char volatile *_Target, char _Value); 232static __inline__ 233long __cdecl _InterlockedExchangeAdd(long volatile *_Addend, long _Value); 234long _InterlockedExchangeAdd_HLEAcquire(long volatile *, long); 235long _InterlockedExchangeAdd_HLERelease(long volatile *, long); 236static __inline__ 237short _InterlockedExchangeAdd16(short volatile *_Addend, short _Value); 238__int64 _InterlockedExchangeAdd64_HLEAcquire(__int64 volatile *, __int64); 239__int64 _InterlockedExchangeAdd64_HLERelease(__int64 volatile *, __int64); 240static __inline__ 241char _InterlockedExchangeAdd8(char volatile *_Addend, char _Value); 242static __inline__ 243long __cdecl _InterlockedIncrement(long volatile *_Addend); 244static __inline__ 245short _InterlockedIncrement16(short volatile *_Addend); 246static __inline__ 247long _InterlockedOr(long volatile *_Value, long _Mask); 248static __inline__ 249short _InterlockedOr16(short volatile *_Value, short _Mask); 250static __inline__ 251char _InterlockedOr8(char volatile *_Value, char _Mask); 252static __inline__ 253long _InterlockedXor(long volatile *_Value, long _Mask); 254static __inline__ 255short _InterlockedXor16(short volatile *_Value, short _Mask); 256static __inline__ 257char _InterlockedXor8(char volatile *_Value, char _Mask); 258void __cdecl _invpcid(unsigned int, void *); 259static __inline__ 260unsigned long __cdecl _lrotl(unsigned long, int); 261static __inline__ 262unsigned long __cdecl _lrotr(unsigned long, int); 263static __inline__ 264unsigned int _lzcnt_u32(unsigned int); 265static __inline__ 266void _ReadBarrier(void); 267static __inline__ 268void _ReadWriteBarrier(void); 269static __inline__ 270void *_ReturnAddress(void); 271unsigned int _rorx_u32(unsigned int, const unsigned int); 272int __cdecl _rdrand16_step(unsigned short *); 273int __cdecl _rdrand32_step(unsigned int *); 274static __inline__ 275unsigned int __cdecl _rotl(unsigned int _Value, int _Shift); 276static __inline__ 277unsigned short _rotl16(unsigned short _Value, unsigned char _Shift); 278static __inline__ 279unsigned __int64 __cdecl _rotl64(unsigned __int64 _Value, int _Shift); 280static __inline__ 281unsigned char _rotl8(unsigned char _Value, unsigned char _Shift); 282static __inline__ 283unsigned int __cdecl _rotr(unsigned int _Value, int _Shift); 284static __inline__ 285unsigned short _rotr16(unsigned short _Value, unsigned char _Shift); 286static __inline__ 287unsigned __int64 __cdecl _rotr64(unsigned __int64 _Value, int _Shift); 288static __inline__ 289unsigned char _rotr8(unsigned char _Value, unsigned char _Shift); 290int _sarx_i32(int, unsigned int); 291int __cdecl _setjmp(jmp_buf); 292unsigned int _shlx_u32(unsigned int, unsigned int); 293unsigned int _shrx_u32(unsigned int, unsigned int); 294void _Store_HLERelease(long volatile *, long); 295void _Store64_HLERelease(__int64 volatile *, __int64); 296void _StorePointer_HLERelease(void *volatile *, void *); 297unsigned int _t1mskc_u32(unsigned int); 298unsigned int _tzcnt_u32(unsigned int); 299unsigned int _tzmsk_u32(unsigned int); 300static __inline__ 301void _WriteBarrier(void); 302void _xabort(const unsigned int imm); 303unsigned __int32 xbegin(void); 304void _xend(void); 305static __inline__ 306unsigned __int64 __cdecl _xgetbv(unsigned int); 307void __cdecl _xrstor(void const *, unsigned __int64); 308void __cdecl _xsave(void *, unsigned __int64); 309void __cdecl _xsaveopt(void *, unsigned __int64); 310void __cdecl _xsetbv(unsigned int, unsigned __int64); 311unsigned char _xtest(void); 312 313/* These additional intrinsics are turned on in x64/amd64/x86_64 mode. */ 314#ifdef __x86_64__ 315void __addgsbyte(unsigned long, unsigned char); 316void __addgsdword(unsigned long, unsigned long); 317void __addgsqword(unsigned long, unsigned __int64); 318void __addgsword(unsigned long, unsigned short); 319static __inline__ 320void __faststorefence(void); 321void __incgsbyte(unsigned long); 322void __incgsdword(unsigned long); 323void __incgsqword(unsigned long); 324void __incgsword(unsigned long); 325unsigned char __lwpins64(unsigned __int64, unsigned int, unsigned int); 326void __lwpval64(unsigned __int64, unsigned int, unsigned int); 327unsigned __int64 __lzcnt64(unsigned __int64); 328static __inline__ 329void __movsq(unsigned long long *, unsigned long long const *, size_t); 330__int64 __mulh(__int64, __int64); 331static __inline__ 332unsigned __int64 __popcnt64(unsigned __int64); 333static __inline__ 334unsigned char __readgsbyte(unsigned long); 335static __inline__ 336unsigned long __readgsdword(unsigned long); 337static __inline__ 338unsigned __int64 __readgsqword(unsigned long); 339unsigned short __readgsword(unsigned long); 340unsigned __int64 __shiftleft128(unsigned __int64 _LowPart, 341 unsigned __int64 _HighPart, 342 unsigned char _Shift); 343unsigned __int64 __shiftright128(unsigned __int64 _LowPart, 344 unsigned __int64 _HighPart, 345 unsigned char _Shift); 346static __inline__ 347void __stosq(unsigned __int64 *, unsigned __int64, size_t); 348unsigned __int64 __umulh(unsigned __int64, unsigned __int64); 349unsigned char __vmx_on(unsigned __int64 *); 350unsigned char __vmx_vmclear(unsigned __int64 *); 351unsigned char __vmx_vmlaunch(void); 352unsigned char __vmx_vmptrld(unsigned __int64 *); 353unsigned char __vmx_vmread(size_t, size_t *); 354unsigned char __vmx_vmresume(void); 355unsigned char __vmx_vmwrite(size_t, size_t); 356void __writegsbyte(unsigned long, unsigned char); 357void __writegsdword(unsigned long, unsigned long); 358void __writegsqword(unsigned long, unsigned __int64); 359void __writegsword(unsigned long, unsigned short); 360unsigned __int64 _andn_u64(unsigned __int64, unsigned __int64); 361unsigned __int64 _bextr_u64(unsigned __int64, unsigned int, unsigned int); 362unsigned __int64 _bextri_u64(unsigned __int64, unsigned int); 363static __inline__ 364unsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask); 365static __inline__ 366unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask); 367static __inline__ 368unsigned char _bittest64(__int64 const *, __int64); 369static __inline__ 370unsigned char _bittestandcomplement64(__int64 *, __int64); 371static __inline__ 372unsigned char _bittestandreset64(__int64 *, __int64); 373static __inline__ 374unsigned char _bittestandset64(__int64 *, __int64); 375unsigned __int64 _blcfill_u64(unsigned __int64); 376unsigned __int64 _blci_u64(unsigned __int64); 377unsigned __int64 _blcic_u64(unsigned __int64); 378unsigned __int64 _blcmsk_u64(unsigned __int64); 379unsigned __int64 _blcs_u64(unsigned __int64); 380unsigned __int64 _blsfill_u64(unsigned __int64); 381unsigned __int64 _blsi_u64(unsigned __int64); 382unsigned __int64 _blsic_u64(unsigned __int64); 383unsigned __int64 _blsmsk_u64(unsigned __int64); 384unsigned __int64 _blsr_u64(unsigned __int64); 385unsigned __int64 __cdecl _byteswap_uint64(unsigned __int64); 386unsigned __int64 _bzhi_u64(unsigned __int64, unsigned int); 387void __cdecl _fxrstor64(void const *); 388void __cdecl _fxsave64(void *); 389long _InterlockedAnd_np(long volatile *_Value, long _Mask); 390short _InterlockedAnd16_np(short volatile *_Value, short _Mask); 391__int64 _InterlockedAnd64_np(__int64 volatile *_Value, __int64 _Mask); 392char _InterlockedAnd8_np(char volatile *_Value, char _Mask); 393unsigned char _interlockedbittestandreset64(__int64 volatile *, __int64); 394static __inline__ 395unsigned char _interlockedbittestandset64(__int64 volatile *, __int64); 396long _InterlockedCompareExchange_np(long volatile *_Destination, long _Exchange, 397 long _Comparand); 398unsigned char _InterlockedCompareExchange128(__int64 volatile *_Destination, 399 __int64 _ExchangeHigh, 400 __int64 _ExchangeLow, 401 __int64 *_CompareandResult); 402unsigned char _InterlockedCompareExchange128_np(__int64 volatile *_Destination, 403 __int64 _ExchangeHigh, 404 __int64 _ExchangeLow, 405 __int64 *_ComparandResult); 406short _InterlockedCompareExchange16_np(short volatile *_Destination, 407 short _Exchange, short _Comparand); 408__int64 _InterlockedCompareExchange64_HLEAcquire(__int64 volatile *, __int64, 409 __int64); 410__int64 _InterlockedCompareExchange64_HLERelease(__int64 volatile *, __int64, 411 __int64); 412__int64 _InterlockedCompareExchange64_np(__int64 volatile *_Destination, 413 __int64 _Exchange, __int64 _Comparand); 414static __inline__ 415void *_InterlockedCompareExchangePointer(void *volatile *_Destination, 416 void *_Exchange, void *_Comparand); 417void *_InterlockedCompareExchangePointer_np(void *volatile *_Destination, 418 void *_Exchange, void *_Comparand); 419static __inline__ 420__int64 _InterlockedDecrement64(__int64 volatile *_Addend); 421static __inline__ 422__int64 _InterlockedExchange64(__int64 volatile *_Target, __int64 _Value); 423static __inline__ 424__int64 _InterlockedExchangeAdd64(__int64 volatile *_Addend, __int64 _Value); 425static __inline__ 426void *_InterlockedExchangePointer(void *volatile *_Target, void *_Value); 427static __inline__ 428__int64 _InterlockedIncrement64(__int64 volatile *_Addend); 429long _InterlockedOr_np(long volatile *_Value, long _Mask); 430short _InterlockedOr16_np(short volatile *_Value, short _Mask); 431static __inline__ 432__int64 _InterlockedOr64(__int64 volatile *_Value, __int64 _Mask); 433__int64 _InterlockedOr64_np(__int64 volatile *_Value, __int64 _Mask); 434char _InterlockedOr8_np(char volatile *_Value, char _Mask); 435long _InterlockedXor_np(long volatile *_Value, long _Mask); 436short _InterlockedXor16_np(short volatile *_Value, short _Mask); 437static __inline__ 438__int64 _InterlockedXor64(__int64 volatile *_Value, __int64 _Mask); 439__int64 _InterlockedXor64_np(__int64 volatile *_Value, __int64 _Mask); 440char _InterlockedXor8_np(char volatile *_Value, char _Mask); 441static __inline__ 442unsigned __int64 _lzcnt_u64(unsigned __int64); 443__int64 _mul128(__int64 _Multiplier, __int64 _Multiplicand, 444 __int64 *_HighProduct); 445unsigned int __cdecl _readfsbase_u32(void); 446unsigned __int64 __cdecl _readfsbase_u64(void); 447unsigned int __cdecl _readgsbase_u32(void); 448unsigned __int64 __cdecl _readgsbase_u64(void); 449unsigned __int64 _rorx_u64(unsigned __int64, const unsigned int); 450__int64 _sarx_i64(__int64, unsigned int); 451/* FIXME: Need definition for jmp_buf. 452 int __cdecl _setjmpex(jmp_buf); */ 453unsigned __int64 _shlx_u64(unsigned __int64, unsigned int); 454unsigned __int64 shrx_u64(unsigned __int64, unsigned int); 455unsigned __int64 _tzcnt_u64(unsigned __int64); 456unsigned __int64 _tzmsk_u64(unsigned __int64); 457unsigned __int64 _umul128(unsigned __int64 _Multiplier, 458 unsigned __int64 _Multiplicand, 459 unsigned __int64 *_HighProduct); 460void __cdecl _writefsbase_u32(unsigned int); 461void _cdecl _writefsbase_u64(unsigned __int64); 462void __cdecl _writegsbase_u32(unsigned int); 463void __cdecl _writegsbase_u64(unsigned __int64); 464void __cdecl _xrstor64(void const *, unsigned __int64); 465void __cdecl _xsave64(void *, unsigned __int64); 466void __cdecl _xsaveopt64(void *, unsigned __int64); 467 468#endif /* __x86_64__ */ 469 470/*----------------------------------------------------------------------------*\ 471|* Bit Twiddling 472\*----------------------------------------------------------------------------*/ 473static __inline__ unsigned char __attribute__((__always_inline__, __nodebug__)) 474_rotl8(unsigned char _Value, unsigned char _Shift) { 475 _Shift &= 0x7; 476 return _Shift ? (_Value << _Shift) | (_Value >> (8 - _Shift)) : _Value; 477} 478static __inline__ unsigned char __attribute__((__always_inline__, __nodebug__)) 479_rotr8(unsigned char _Value, unsigned char _Shift) { 480 _Shift &= 0x7; 481 return _Shift ? (_Value >> _Shift) | (_Value << (8 - _Shift)) : _Value; 482} 483static __inline__ unsigned short __attribute__((__always_inline__, __nodebug__)) 484_rotl16(unsigned short _Value, unsigned char _Shift) { 485 _Shift &= 0xf; 486 return _Shift ? (_Value << _Shift) | (_Value >> (16 - _Shift)) : _Value; 487} 488static __inline__ unsigned short __attribute__((__always_inline__, __nodebug__)) 489_rotr16(unsigned short _Value, unsigned char _Shift) { 490 _Shift &= 0xf; 491 return _Shift ? (_Value >> _Shift) | (_Value << (16 - _Shift)) : _Value; 492} 493static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__)) 494_rotl(unsigned int _Value, int _Shift) { 495 _Shift &= 0x1f; 496 return _Shift ? (_Value << _Shift) | (_Value >> (32 - _Shift)) : _Value; 497} 498static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__)) 499_rotr(unsigned int _Value, int _Shift) { 500 _Shift &= 0x1f; 501 return _Shift ? (_Value >> _Shift) | (_Value << (32 - _Shift)) : _Value; 502} 503static __inline__ unsigned long __attribute__((__always_inline__, __nodebug__)) 504_lrotl(unsigned long _Value, int _Shift) { 505 _Shift &= 0x1f; 506 return _Shift ? (_Value << _Shift) | (_Value >> (32 - _Shift)) : _Value; 507} 508static __inline__ unsigned long __attribute__((__always_inline__, __nodebug__)) 509_lrotr(unsigned long _Value, int _Shift) { 510 _Shift &= 0x1f; 511 return _Shift ? (_Value >> _Shift) | (_Value << (32 - _Shift)) : _Value; 512} 513static 514__inline__ unsigned __int64 __attribute__((__always_inline__, __nodebug__)) 515_rotl64(unsigned __int64 _Value, int _Shift) { 516 _Shift &= 0x3f; 517 return _Shift ? (_Value << _Shift) | (_Value >> (64 - _Shift)) : _Value; 518} 519static 520__inline__ unsigned __int64 __attribute__((__always_inline__, __nodebug__)) 521_rotr64(unsigned __int64 _Value, int _Shift) { 522 _Shift &= 0x3f; 523 return _Shift ? (_Value >> _Shift) | (_Value << (64 - _Shift)) : _Value; 524} 525/*----------------------------------------------------------------------------*\ 526|* Bit Counting and Testing 527\*----------------------------------------------------------------------------*/ 528static __inline__ unsigned char __attribute__((__always_inline__, __nodebug__)) 529_BitScanForward(unsigned long *_Index, unsigned long _Mask) { 530 if (!_Mask) 531 return 0; 532 *_Index = __builtin_ctzl(_Mask); 533 return 1; 534} 535static __inline__ unsigned char __attribute__((__always_inline__, __nodebug__)) 536_BitScanReverse(unsigned long *_Index, unsigned long _Mask) { 537 if (!_Mask) 538 return 0; 539 *_Index = 31 - __builtin_clzl(_Mask); 540 return 1; 541} 542static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__)) 543_lzcnt_u32(unsigned int a) { 544 if (!a) 545 return 32; 546 return __builtin_clzl(a); 547} 548static __inline__ unsigned short __attribute__((__always_inline__, __nodebug__)) 549__popcnt16(unsigned short value) { 550 return __builtin_popcount((int)value); 551} 552static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__)) 553__popcnt(unsigned int value) { 554 return __builtin_popcount(value); 555} 556static __inline__ unsigned char __attribute__((__always_inline__, __nodebug__)) 557_bittest(long const *a, long b) { 558 return (*a >> b) & 1; 559} 560static __inline__ unsigned char __attribute__((__always_inline__, __nodebug__)) 561_bittestandcomplement(long *a, long b) { 562 unsigned char x = (*a >> b) & 1; 563 *a = *a ^ (1 << b); 564 return x; 565} 566static __inline__ unsigned char __attribute__((__always_inline__, __nodebug__)) 567_bittestandreset(long *a, long b) { 568 unsigned char x = (*a >> b) & 1; 569 *a = *a & ~(1 << b); 570 return x; 571} 572static __inline__ unsigned char __attribute__((__always_inline__, __nodebug__)) 573_bittestandset(long *a, long b) { 574 unsigned char x = (*a >> b) & 1; 575 *a = *a | (1 << b); 576 return x; 577} 578static __inline__ unsigned char __attribute__((__always_inline__, __nodebug__)) 579_interlockedbittestandset(long volatile *__BitBase, long __BitPos) { 580 unsigned char __Res; 581 __asm__ ("xor %0, %0\n" 582 "lock bts %2, %1\n" 583 "setc %0\n" 584 : "=r" (__Res), "+m"(*__BitBase) 585 : "Ir"(__BitPos)); 586 return __Res; 587} 588#ifdef __x86_64__ 589static __inline__ unsigned char __attribute__((__always_inline__, __nodebug__)) 590_BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask) { 591 if (!_Mask) 592 return 0; 593 *_Index = __builtin_ctzll(_Mask); 594 return 1; 595} 596static __inline__ unsigned char __attribute__((__always_inline__, __nodebug__)) 597_BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask) { 598 if (!_Mask) 599 return 0; 600 *_Index = 63 - __builtin_clzll(_Mask); 601 return 1; 602} 603static 604__inline__ unsigned __int64 __attribute__((__always_inline__, __nodebug__)) 605_lzcnt_u64(unsigned __int64 a) { 606 if (!a) 607 return 64; 608 return __builtin_clzll(a); 609} 610static __inline__ 611unsigned __int64 __attribute__((__always_inline__, __nodebug__)) 612 __popcnt64(unsigned __int64 value) { 613 return __builtin_popcountll(value); 614} 615static __inline__ unsigned char __attribute__((__always_inline__, __nodebug__)) 616_bittest64(__int64 const *a, __int64 b) { 617 return (*a >> b) & 1; 618} 619static __inline__ unsigned char __attribute__((__always_inline__, __nodebug__)) 620_bittestandcomplement64(__int64 *a, __int64 b) { 621 unsigned char x = (*a >> b) & 1; 622 *a = *a ^ (1ll << b); 623 return x; 624} 625static __inline__ unsigned char __attribute__((__always_inline__, __nodebug__)) 626_bittestandreset64(__int64 *a, __int64 b) { 627 unsigned char x = (*a >> b) & 1; 628 *a = *a & ~(1ll << b); 629 return x; 630} 631static __inline__ unsigned char __attribute__((__always_inline__, __nodebug__)) 632_bittestandset64(__int64 *a, __int64 b) { 633 unsigned char x = (*a >> b) & 1; 634 *a = *a | (1ll << b); 635 return x; 636} 637static __inline__ unsigned char __attribute__((__always_inline__, __nodebug__)) 638_interlockedbittestandset64(__int64 volatile *__BitBase, __int64 __BitPos) { 639 unsigned char __Res; 640 __asm__ ("xor %0, %0\n" 641 "lock bts %2, %1\n" 642 "setc %0\n" 643 : "=r" (__Res), "+m"(*__BitBase) 644 : "Ir"(__BitPos)); 645 return __Res; 646} 647#endif 648/*----------------------------------------------------------------------------*\ 649|* Interlocked Exchange Add 650\*----------------------------------------------------------------------------*/ 651static __inline__ char __attribute__((__always_inline__, __nodebug__)) 652_InterlockedExchangeAdd8(char volatile *_Addend, char _Value) { 653 return __atomic_add_fetch(_Addend, _Value, 0) - _Value; 654} 655static __inline__ short __attribute__((__always_inline__, __nodebug__)) 656_InterlockedExchangeAdd16(short volatile *_Addend, short _Value) { 657 return __atomic_add_fetch(_Addend, _Value, 0) - _Value; 658} 659#ifdef __x86_64__ 660static __inline__ __int64 __attribute__((__always_inline__, __nodebug__)) 661_InterlockedExchangeAdd64(__int64 volatile *_Addend, __int64 _Value) { 662 return __atomic_add_fetch(_Addend, _Value, 0) - _Value; 663} 664#endif 665/*----------------------------------------------------------------------------*\ 666|* Interlocked Exchange Sub 667\*----------------------------------------------------------------------------*/ 668static __inline__ char __attribute__((__always_inline__, __nodebug__)) 669_InterlockedExchangeSub8(char volatile *_Subend, char _Value) { 670 return __atomic_sub_fetch(_Subend, _Value, 0) + _Value; 671} 672static __inline__ short __attribute__((__always_inline__, __nodebug__)) 673_InterlockedExchangeSub16(short volatile *_Subend, short _Value) { 674 return __atomic_sub_fetch(_Subend, _Value, 0) + _Value; 675} 676static __inline__ long __attribute__((__always_inline__, __nodebug__)) 677_InterlockedExchangeSub(long volatile *_Subend, long _Value) { 678 return __atomic_sub_fetch(_Subend, _Value, 0) + _Value; 679} 680#ifdef __x86_64__ 681static __inline__ __int64 __attribute__((__always_inline__, __nodebug__)) 682_InterlockedExchangeSub64(__int64 volatile *_Subend, __int64 _Value) { 683 return __atomic_sub_fetch(_Subend, _Value, 0) + _Value; 684} 685#endif 686/*----------------------------------------------------------------------------*\ 687|* Interlocked Increment 688\*----------------------------------------------------------------------------*/ 689static __inline__ short __attribute__((__always_inline__, __nodebug__)) 690_InterlockedIncrement16(short volatile *_Value) { 691 return __atomic_add_fetch(_Value, 1, 0); 692} 693#ifdef __x86_64__ 694static __inline__ __int64 __attribute__((__always_inline__, __nodebug__)) 695_InterlockedIncrement64(__int64 volatile *_Value) { 696 return __atomic_add_fetch(_Value, 1, 0); 697} 698#endif 699/*----------------------------------------------------------------------------*\ 700|* Interlocked Decrement 701\*----------------------------------------------------------------------------*/ 702static __inline__ short __attribute__((__always_inline__, __nodebug__)) 703_InterlockedDecrement16(short volatile *_Value) { 704 return __atomic_sub_fetch(_Value, 1, 0); 705} 706#ifdef __x86_64__ 707static __inline__ __int64 __attribute__((__always_inline__, __nodebug__)) 708_InterlockedDecrement64(__int64 volatile *_Value) { 709 return __atomic_sub_fetch(_Value, 1, 0); 710} 711#endif 712/*----------------------------------------------------------------------------*\ 713|* Interlocked And 714\*----------------------------------------------------------------------------*/ 715static __inline__ char __attribute__((__always_inline__, __nodebug__)) 716_InterlockedAnd8(char volatile *_Value, char _Mask) { 717 return __atomic_and_fetch(_Value, _Mask, 0); 718} 719static __inline__ short __attribute__((__always_inline__, __nodebug__)) 720_InterlockedAnd16(short volatile *_Value, short _Mask) { 721 return __atomic_and_fetch(_Value, _Mask, 0); 722} 723static __inline__ long __attribute__((__always_inline__, __nodebug__)) 724_InterlockedAnd(long volatile *_Value, long _Mask) { 725 return __atomic_and_fetch(_Value, _Mask, 0); 726} 727#ifdef __x86_64__ 728static __inline__ __int64 __attribute__((__always_inline__, __nodebug__)) 729_InterlockedAnd64(__int64 volatile *_Value, __int64 _Mask) { 730 return __atomic_and_fetch(_Value, _Mask, 0); 731} 732#endif 733/*----------------------------------------------------------------------------*\ 734|* Interlocked Or 735\*----------------------------------------------------------------------------*/ 736static __inline__ char __attribute__((__always_inline__, __nodebug__)) 737_InterlockedOr8(char volatile *_Value, char _Mask) { 738 return __atomic_or_fetch(_Value, _Mask, 0); 739} 740static __inline__ short __attribute__((__always_inline__, __nodebug__)) 741_InterlockedOr16(short volatile *_Value, short _Mask) { 742 return __atomic_or_fetch(_Value, _Mask, 0); 743} 744static __inline__ long __attribute__((__always_inline__, __nodebug__)) 745_InterlockedOr(long volatile *_Value, long _Mask) { 746 return __atomic_or_fetch(_Value, _Mask, 0); 747} 748#ifdef __x86_64__ 749static __inline__ __int64 __attribute__((__always_inline__, __nodebug__)) 750_InterlockedOr64(__int64 volatile *_Value, __int64 _Mask) { 751 return __atomic_or_fetch(_Value, _Mask, 0); 752} 753#endif 754/*----------------------------------------------------------------------------*\ 755|* Interlocked Xor 756\*----------------------------------------------------------------------------*/ 757static __inline__ char __attribute__((__always_inline__, __nodebug__)) 758_InterlockedXor8(char volatile *_Value, char _Mask) { 759 return __atomic_xor_fetch(_Value, _Mask, 0); 760} 761static __inline__ short __attribute__((__always_inline__, __nodebug__)) 762_InterlockedXor16(short volatile *_Value, short _Mask) { 763 return __atomic_xor_fetch(_Value, _Mask, 0); 764} 765static __inline__ long __attribute__((__always_inline__, __nodebug__)) 766_InterlockedXor(long volatile *_Value, long _Mask) { 767 return __atomic_xor_fetch(_Value, _Mask, 0); 768} 769#ifdef __x86_64__ 770static __inline__ __int64 __attribute__((__always_inline__, __nodebug__)) 771_InterlockedXor64(__int64 volatile *_Value, __int64 _Mask) { 772 return __atomic_xor_fetch(_Value, _Mask, 0); 773} 774#endif 775/*----------------------------------------------------------------------------*\ 776|* Interlocked Exchange 777\*----------------------------------------------------------------------------*/ 778static __inline__ char __attribute__((__always_inline__, __nodebug__)) 779_InterlockedExchange8(char volatile *_Target, char _Value) { 780 __atomic_exchange(_Target, &_Value, &_Value, 0); 781 return _Value; 782} 783static __inline__ short __attribute__((__always_inline__, __nodebug__)) 784_InterlockedExchange16(short volatile *_Target, short _Value) { 785 __atomic_exchange(_Target, &_Value, &_Value, 0); 786 return _Value; 787} 788static __inline__ long __attribute__((__always_inline__, __nodebug__)) 789_InterlockedExchange(long volatile *_Target, long _Value) { 790 __atomic_exchange(_Target, &_Value, &_Value, 0); 791 return _Value; 792} 793#ifdef __x86_64__ 794static __inline__ __int64 __attribute__((__always_inline__, __nodebug__)) 795_InterlockedExchange64(__int64 volatile *_Target, __int64 _Value) { 796 __atomic_exchange(_Target, &_Value, &_Value, 0); 797 return _Value; 798} 799static __inline__ void *__attribute__((__always_inline__, __nodebug__)) 800_InterlockedExchangePointer(void *volatile *_Target, void *_Value) { 801 __atomic_exchange(_Target, &_Value, &_Value, 0); 802 return _Value; 803} 804#endif 805/*----------------------------------------------------------------------------*\ 806|* Interlocked Compare Exchange 807\*----------------------------------------------------------------------------*/ 808static __inline__ char __attribute__((__always_inline__, __nodebug__)) 809_InterlockedCompareExchange8(char volatile *_Destination, 810 char _Exchange, char _Comparand) { 811 __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, 0, 0); 812 return _Comparand; 813} 814static __inline__ short __attribute__((__always_inline__, __nodebug__)) 815_InterlockedCompareExchange16(short volatile *_Destination, 816 short _Exchange, short _Comparand) { 817 __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, 0, 0); 818 return _Comparand; 819} 820#ifdef __x86_64__ 821static __inline__ void *__attribute__((__always_inline__, __nodebug__)) 822_InterlockedCompareExchangePointer(void *volatile *_Destination, 823 void *_Exchange, void *_Comparand) { 824 __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, 0, 0); 825 return _Comparand; 826} 827#endif 828#ifdef __x86_64__ 829static __inline__ __int64 __attribute__((__always_inline__, __nodebug__)) 830_InterlockedCompareExchange64(__int64 volatile *_Destination, 831 __int64 _Exchange, __int64 _Comparand) { 832 __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, 0, 0); 833 return _Comparand; 834} 835#endif 836/*----------------------------------------------------------------------------*\ 837|* Barriers 838\*----------------------------------------------------------------------------*/ 839static __inline__ void __attribute__((__always_inline__, __nodebug__)) 840__attribute__((deprecated("use other intrinsics or C++11 atomics instead"))) 841_ReadWriteBarrier(void) { 842 __asm__ volatile ("" : : : "memory"); 843} 844static __inline__ void __attribute__((__always_inline__, __nodebug__)) 845__attribute__((deprecated("use other intrinsics or C++11 atomics instead"))) 846_ReadBarrier(void) { 847 __asm__ volatile ("" : : : "memory"); 848} 849static __inline__ void __attribute__((__always_inline__, __nodebug__)) 850__attribute__((deprecated("use other intrinsics or C++11 atomics instead"))) 851_WriteBarrier(void) { 852 __asm__ volatile ("" : : : "memory"); 853} 854#ifdef __x86_64__ 855static __inline__ void __attribute__((__always_inline__, __nodebug__)) 856__faststorefence(void) { 857 __asm__ volatile("lock orq $0, (%%rsp)" : : : "memory"); 858} 859#endif 860/*----------------------------------------------------------------------------*\ 861|* readfs, readgs 862|* (Pointers in address space #256 and #257 are relative to the GS and FS 863|* segment registers, respectively.) 864\*----------------------------------------------------------------------------*/ 865#define __ptr_to_addr_space(__addr_space_nbr, __type, __offset) \ 866 ((volatile __type __attribute__((__address_space__(__addr_space_nbr)))*) \ 867 (__offset)) 868 869#ifdef __i386__ 870static __inline__ unsigned char __attribute__((__always_inline__, __nodebug__)) 871__readfsbyte(unsigned long __offset) { 872 return *__ptr_to_addr_space(257, unsigned char, __offset); 873} 874static __inline__ unsigned long __attribute__((__always_inline__, __nodebug__)) 875__readfsdword(unsigned long __offset) { 876 return *__ptr_to_addr_space(257, unsigned long, __offset); 877} 878static __inline__ unsigned __int64 __attribute__((__always_inline__, __nodebug__)) 879__readfsqword(unsigned long __offset) { 880 return *__ptr_to_addr_space(257, unsigned __int64, __offset); 881} 882static __inline__ unsigned short __attribute__((__always_inline__, __nodebug__)) 883__readfsword(unsigned long __offset) { 884 return *__ptr_to_addr_space(257, unsigned short, __offset); 885} 886#endif 887#ifdef __x86_64__ 888static __inline__ unsigned char __attribute__((__always_inline__, __nodebug__)) 889__readgsbyte(unsigned long __offset) { 890 return *__ptr_to_addr_space(256, unsigned char, __offset); 891} 892static __inline__ unsigned long __attribute__((__always_inline__, __nodebug__)) 893__readgsdword(unsigned long __offset) { 894 return *__ptr_to_addr_space(256, unsigned long, __offset); 895} 896static __inline__ unsigned __int64 __attribute__((__always_inline__, __nodebug__)) 897__readgsqword(unsigned long __offset) { 898 return *__ptr_to_addr_space(256, unsigned __int64, __offset); 899} 900static __inline__ unsigned short __attribute__((__always_inline__, __nodebug__)) 901__readgsword(unsigned long __offset) { 902 return *__ptr_to_addr_space(256, unsigned short, __offset); 903} 904#endif 905#undef __ptr_to_addr_space 906/*----------------------------------------------------------------------------*\ 907|* movs, stos 908\*----------------------------------------------------------------------------*/ 909static __inline__ void __attribute__((__always_inline__, __nodebug__)) 910__movsb(unsigned char *__dst, unsigned char const *__src, size_t __n) { 911 __asm__("rep movsb" : : "D"(__dst), "S"(__src), "c"(__n) 912 : "%edi", "%esi", "%ecx"); 913} 914static __inline__ void __attribute__((__always_inline__, __nodebug__)) 915__movsd(unsigned long *__dst, unsigned long const *__src, size_t __n) { 916 __asm__("rep movsl" : : "D"(__dst), "S"(__src), "c"(__n) 917 : "%edi", "%esi", "%ecx"); 918} 919static __inline__ void __attribute__((__always_inline__, __nodebug__)) 920__movsw(unsigned short *__dst, unsigned short const *__src, size_t __n) { 921 __asm__("rep movsh" : : "D"(__dst), "S"(__src), "c"(__n) 922 : "%edi", "%esi", "%ecx"); 923} 924static __inline__ void __attribute__((__always_inline__, __nodebug__)) 925__stosb(unsigned char *__dst, unsigned char __x, size_t __n) { 926 __asm__("rep stosb" : : "D"(__dst), "a"(__x), "c"(__n) 927 : "%edi", "%ecx"); 928} 929static __inline__ void __attribute__((__always_inline__, __nodebug__)) 930__stosd(unsigned long *__dst, unsigned long __x, size_t __n) { 931 __asm__("rep stosl" : : "D"(__dst), "a"(__x), "c"(__n) 932 : "%edi", "%ecx"); 933} 934static __inline__ void __attribute__((__always_inline__, __nodebug__)) 935__stosw(unsigned short *__dst, unsigned short __x, size_t __n) { 936 __asm__("rep stosh" : : "D"(__dst), "a"(__x), "c"(__n) 937 : "%edi", "%ecx"); 938} 939#ifdef __x86_64__ 940static __inline__ void __attribute__((__always_inline__, __nodebug__)) 941__movsq(unsigned long long *__dst, unsigned long long const *__src, size_t __n) { 942 __asm__("rep movsq" : : "D"(__dst), "S"(__src), "c"(__n) 943 : "%edi", "%esi", "%ecx"); 944} 945static __inline__ void __attribute__((__always_inline__, __nodebug__)) 946__stosq(unsigned __int64 *__dst, unsigned __int64 __x, size_t __n) { 947 __asm__("rep stosq" : : "D"(__dst), "a"(__x), "c"(__n) 948 : "%edi", "%ecx"); 949} 950#endif 951 952/*----------------------------------------------------------------------------*\ 953|* Misc 954\*----------------------------------------------------------------------------*/ 955static __inline__ void * __attribute__((__always_inline__, __nodebug__)) 956_AddressOfReturnAddress(void) { 957 return (void*)((char*)__builtin_frame_address(0) + sizeof(void*)); 958} 959static __inline__ void * __attribute__((__always_inline__, __nodebug__)) 960_ReturnAddress(void) { 961 return __builtin_return_address(0); 962} 963static __inline__ void __attribute__((__always_inline__, __nodebug__)) 964__cpuid(int __info[4], int __level) { 965 __asm__ ("cpuid" : "=a"(__info[0]), "=b" (__info[1]), "=c"(__info[2]), "=d"(__info[3]) 966 : "a"(__level)); 967} 968static __inline__ void __attribute__((__always_inline__, __nodebug__)) 969__cpuidex(int __info[4], int __level, int __ecx) { 970 __asm__ ("cpuid" : "=a"(__info[0]), "=b" (__info[1]), "=c"(__info[2]), "=d"(__info[3]) 971 : "a"(__level), "c"(__ecx)); 972} 973static __inline__ unsigned __int64 __cdecl __attribute__((__always_inline__, __nodebug__)) 974_xgetbv(unsigned int __xcr_no) { 975 unsigned int __eax, __edx; 976 __asm__ ("xgetbv" : "=a" (__eax), "=d" (__edx) : "c" (__xcr_no)); 977 return ((unsigned __int64)__edx << 32) | __eax; 978} 979static __inline__ void __attribute__((__always_inline__, __nodebug__)) 980__halt(void) { 981 __asm__ volatile ("hlt"); 982} 983 984#ifdef __cplusplus 985} 986#endif 987 988#endif /* __INTRIN_H */ 989#endif /* _MSC_VER */ 990