15c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper/*===---- bmiintrin.h - BMI intrinsics -------------------------------------=== 25c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper * 35c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper * Permission is hereby granted, free of charge, to any person obtaining a copy 45c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper * of this software and associated documentation files (the "Software"), to deal 55c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper * in the Software without restriction, including without limitation the rights 65c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 75c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper * copies of the Software, and to permit persons to whom the Software is 85c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper * furnished to do so, subject to the following conditions: 95c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper * 105c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper * The above copyright notice and this permission notice shall be included in 115c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper * all copies or substantial portions of the Software. 125c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper * 135c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 145c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 155c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 165c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 175c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 185c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 195c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper * THE SOFTWARE. 205c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper * 215c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper *===-----------------------------------------------------------------------=== 225c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper */ 235c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper 245c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper#if !defined __X86INTRIN_H && !defined __IMMINTRIN_H 255c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper#error "Never use <bmiintrin.h> directly; include <x86intrin.h> instead." 265c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper#endif 275c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper 285c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper#ifndef __BMI__ 295c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper# error "BMI instruction set not enabled" 305c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper#endif /* __BMI__ */ 315c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper 325c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper#ifndef __BMIINTRIN_H 335c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper#define __BMIINTRIN_H 345c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper 35ef8225444452a1486bd721f3285301fe84643b00Stephen Hines#define _tzcnt_u16(a) (__tzcnt_u16((a))) 36ef8225444452a1486bd721f3285301fe84643b00Stephen Hines#define _andn_u32(a, b) (__andn_u32((a), (b))) 37ef8225444452a1486bd721f3285301fe84643b00Stephen Hines/* _bextr_u32 != __bextr_u32 */ 38ef8225444452a1486bd721f3285301fe84643b00Stephen Hines#define _blsi_u32(a) (__blsi_u32((a))) 39ef8225444452a1486bd721f3285301fe84643b00Stephen Hines#define _blsmsk_u32(a) (__blsmsk_u32((a))) 40ef8225444452a1486bd721f3285301fe84643b00Stephen Hines#define _blsr_u32(a) (__blsr_u32((a))) 41ef8225444452a1486bd721f3285301fe84643b00Stephen Hines#define _tzcnt_u32(a) (__tzcnt_u32((a))) 42ef8225444452a1486bd721f3285301fe84643b00Stephen Hines 435c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topperstatic __inline__ unsigned short __attribute__((__always_inline__, __nodebug__)) 44435d2682893f5ebdaa652b567073ff223f5ada4fCraig Topper__tzcnt_u16(unsigned short __X) 455c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper{ 465c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper return __builtin_ctzs(__X); 475c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper} 485c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper 495c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topperstatic __inline__ unsigned int __attribute__((__always_inline__, __nodebug__)) 500b269c1f3641511fb397488f8b82850416892fc9Craig Topper__andn_u32(unsigned int __X, unsigned int __Y) 510b269c1f3641511fb397488f8b82850416892fc9Craig Topper{ 520b269c1f3641511fb397488f8b82850416892fc9Craig Topper return ~__X & __Y; 530b269c1f3641511fb397488f8b82850416892fc9Craig Topper} 540b269c1f3641511fb397488f8b82850416892fc9Craig Topper 55ef8225444452a1486bd721f3285301fe84643b00Stephen Hines/* AMD-specified, double-leading-underscore version of BEXTR */ 560b269c1f3641511fb397488f8b82850416892fc9Craig Topperstatic __inline__ unsigned int __attribute__((__always_inline__, __nodebug__)) 570b269c1f3641511fb397488f8b82850416892fc9Craig Topper__bextr_u32(unsigned int __X, unsigned int __Y) 580b269c1f3641511fb397488f8b82850416892fc9Craig Topper{ 590b269c1f3641511fb397488f8b82850416892fc9Craig Topper return __builtin_ia32_bextr_u32(__X, __Y); 600b269c1f3641511fb397488f8b82850416892fc9Craig Topper} 610b269c1f3641511fb397488f8b82850416892fc9Craig Topper 62ef8225444452a1486bd721f3285301fe84643b00Stephen Hines/* Intel-specified, single-leading-underscore version of BEXTR */ 63ef8225444452a1486bd721f3285301fe84643b00Stephen Hinesstatic __inline__ unsigned int __attribute__((__always_inline__, __nodebug__)) 64ef8225444452a1486bd721f3285301fe84643b00Stephen Hines_bextr_u32(unsigned int __X, unsigned int __Y, unsigned int __Z) 65ef8225444452a1486bd721f3285301fe84643b00Stephen Hines{ 66ef8225444452a1486bd721f3285301fe84643b00Stephen Hines return __builtin_ia32_bextr_u32 (__X, ((__Y & 0xff) | ((__Z & 0xff) << 8))); 67ef8225444452a1486bd721f3285301fe84643b00Stephen Hines} 68ef8225444452a1486bd721f3285301fe84643b00Stephen Hines 690b269c1f3641511fb397488f8b82850416892fc9Craig Topperstatic __inline__ unsigned int __attribute__((__always_inline__, __nodebug__)) 700b269c1f3641511fb397488f8b82850416892fc9Craig Topper__blsi_u32(unsigned int __X) 710b269c1f3641511fb397488f8b82850416892fc9Craig Topper{ 720b269c1f3641511fb397488f8b82850416892fc9Craig Topper return __X & -__X; 730b269c1f3641511fb397488f8b82850416892fc9Craig Topper} 740b269c1f3641511fb397488f8b82850416892fc9Craig Topper 750b269c1f3641511fb397488f8b82850416892fc9Craig Topperstatic __inline__ unsigned int __attribute__((__always_inline__, __nodebug__)) 760b269c1f3641511fb397488f8b82850416892fc9Craig Topper__blsmsk_u32(unsigned int __X) 770b269c1f3641511fb397488f8b82850416892fc9Craig Topper{ 780b269c1f3641511fb397488f8b82850416892fc9Craig Topper return __X ^ (__X - 1); 790b269c1f3641511fb397488f8b82850416892fc9Craig Topper} 800b269c1f3641511fb397488f8b82850416892fc9Craig Topper 810b269c1f3641511fb397488f8b82850416892fc9Craig Topperstatic __inline__ unsigned int __attribute__((__always_inline__, __nodebug__)) 820b269c1f3641511fb397488f8b82850416892fc9Craig Topper__blsr_u32(unsigned int __X) 830b269c1f3641511fb397488f8b82850416892fc9Craig Topper{ 840b269c1f3641511fb397488f8b82850416892fc9Craig Topper return __X & (__X - 1); 850b269c1f3641511fb397488f8b82850416892fc9Craig Topper} 860b269c1f3641511fb397488f8b82850416892fc9Craig Topper 870b269c1f3641511fb397488f8b82850416892fc9Craig Topperstatic __inline__ unsigned int __attribute__((__always_inline__, __nodebug__)) 88435d2682893f5ebdaa652b567073ff223f5ada4fCraig Topper__tzcnt_u32(unsigned int __X) 895c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper{ 905c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper return __builtin_ctz(__X); 915c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper} 925c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper 935c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper#ifdef __x86_64__ 94ef8225444452a1486bd721f3285301fe84643b00Stephen Hines 95ef8225444452a1486bd721f3285301fe84643b00Stephen Hines#define _andn_u64(a, b) (__andn_u64((a), (b))) 96ef8225444452a1486bd721f3285301fe84643b00Stephen Hines/* _bextr_u64 != __bextr_u64 */ 97ef8225444452a1486bd721f3285301fe84643b00Stephen Hines#define _blsi_u64(a) (__blsi_u64((a))) 98ef8225444452a1486bd721f3285301fe84643b00Stephen Hines#define _blsmsk_u64(a) (__blsmsk_u64((a))) 99ef8225444452a1486bd721f3285301fe84643b00Stephen Hines#define _blsr_u64(a) (__blsr_u64((a))) 100ef8225444452a1486bd721f3285301fe84643b00Stephen Hines#define _tzcnt_u64(a) (__tzcnt_u64((a))) 101ef8225444452a1486bd721f3285301fe84643b00Stephen Hines 1025c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topperstatic __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__)) 1030b269c1f3641511fb397488f8b82850416892fc9Craig Topper__andn_u64 (unsigned long long __X, unsigned long long __Y) 1040b269c1f3641511fb397488f8b82850416892fc9Craig Topper{ 1050b269c1f3641511fb397488f8b82850416892fc9Craig Topper return ~__X & __Y; 1060b269c1f3641511fb397488f8b82850416892fc9Craig Topper} 1070b269c1f3641511fb397488f8b82850416892fc9Craig Topper 108ef8225444452a1486bd721f3285301fe84643b00Stephen Hines/* AMD-specified, double-leading-underscore version of BEXTR */ 1090b269c1f3641511fb397488f8b82850416892fc9Craig Topperstatic __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__)) 1100b269c1f3641511fb397488f8b82850416892fc9Craig Topper__bextr_u64(unsigned long long __X, unsigned long long __Y) 1110b269c1f3641511fb397488f8b82850416892fc9Craig Topper{ 1120b269c1f3641511fb397488f8b82850416892fc9Craig Topper return __builtin_ia32_bextr_u64(__X, __Y); 1130b269c1f3641511fb397488f8b82850416892fc9Craig Topper} 1140b269c1f3641511fb397488f8b82850416892fc9Craig Topper 115ef8225444452a1486bd721f3285301fe84643b00Stephen Hines/* Intel-specified, single-leading-underscore version of BEXTR */ 116ef8225444452a1486bd721f3285301fe84643b00Stephen Hinesstatic __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__)) 117ef8225444452a1486bd721f3285301fe84643b00Stephen Hines_bextr_u64(unsigned long long __X, unsigned int __Y, unsigned int __Z) 118ef8225444452a1486bd721f3285301fe84643b00Stephen Hines{ 119ef8225444452a1486bd721f3285301fe84643b00Stephen Hines return __builtin_ia32_bextr_u64 (__X, ((__Y & 0xff) | ((__Z & 0xff) << 8))); 120ef8225444452a1486bd721f3285301fe84643b00Stephen Hines} 121ef8225444452a1486bd721f3285301fe84643b00Stephen Hines 1220b269c1f3641511fb397488f8b82850416892fc9Craig Topperstatic __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__)) 1230b269c1f3641511fb397488f8b82850416892fc9Craig Topper__blsi_u64(unsigned long long __X) 1240b269c1f3641511fb397488f8b82850416892fc9Craig Topper{ 1250b269c1f3641511fb397488f8b82850416892fc9Craig Topper return __X & -__X; 1260b269c1f3641511fb397488f8b82850416892fc9Craig Topper} 1270b269c1f3641511fb397488f8b82850416892fc9Craig Topper 1280b269c1f3641511fb397488f8b82850416892fc9Craig Topperstatic __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__)) 1290b269c1f3641511fb397488f8b82850416892fc9Craig Topper__blsmsk_u64(unsigned long long __X) 1300b269c1f3641511fb397488f8b82850416892fc9Craig Topper{ 1310b269c1f3641511fb397488f8b82850416892fc9Craig Topper return __X ^ (__X - 1); 1320b269c1f3641511fb397488f8b82850416892fc9Craig Topper} 1330b269c1f3641511fb397488f8b82850416892fc9Craig Topper 1340b269c1f3641511fb397488f8b82850416892fc9Craig Topperstatic __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__)) 1350b269c1f3641511fb397488f8b82850416892fc9Craig Topper__blsr_u64(unsigned long long __X) 1360b269c1f3641511fb397488f8b82850416892fc9Craig Topper{ 1370b269c1f3641511fb397488f8b82850416892fc9Craig Topper return __X & (__X - 1); 1380b269c1f3641511fb397488f8b82850416892fc9Craig Topper} 1390b269c1f3641511fb397488f8b82850416892fc9Craig Topper 1400b269c1f3641511fb397488f8b82850416892fc9Craig Topperstatic __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__)) 141435d2682893f5ebdaa652b567073ff223f5ada4fCraig Topper__tzcnt_u64(unsigned long long __X) 1425c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper{ 1435c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper return __builtin_ctzll(__X); 1445c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper} 145ef8225444452a1486bd721f3285301fe84643b00Stephen Hines 146ef8225444452a1486bd721f3285301fe84643b00Stephen Hines#endif /* __x86_64__ */ 1475c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper 1485c75208a5b88c835bce0a1671015c7e22c72f35fCraig Topper#endif /* __BMIINTRIN_H */ 149