16099914810f6f01c79e5da88794972fa4f1439c4Ying Wang/*===---- bmi2intrin.h - BMI2 intrinsics -----------------------------------=== 26099914810f6f01c79e5da88794972fa4f1439c4Ying Wang * 36099914810f6f01c79e5da88794972fa4f1439c4Ying Wang * Permission is hereby granted, free of charge, to any person obtaining a copy 46099914810f6f01c79e5da88794972fa4f1439c4Ying Wang * of this software and associated documentation files (the "Software"), to deal 56099914810f6f01c79e5da88794972fa4f1439c4Ying Wang * in the Software without restriction, including without limitation the rights 66099914810f6f01c79e5da88794972fa4f1439c4Ying Wang * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 76099914810f6f01c79e5da88794972fa4f1439c4Ying Wang * copies of the Software, and to permit persons to whom the Software is 86099914810f6f01c79e5da88794972fa4f1439c4Ying Wang * furnished to do so, subject to the following conditions: 96099914810f6f01c79e5da88794972fa4f1439c4Ying Wang * 106099914810f6f01c79e5da88794972fa4f1439c4Ying Wang * The above copyright notice and this permission notice shall be included in 116099914810f6f01c79e5da88794972fa4f1439c4Ying Wang * all copies or substantial portions of the Software. 126099914810f6f01c79e5da88794972fa4f1439c4Ying Wang * 136099914810f6f01c79e5da88794972fa4f1439c4Ying Wang * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 146099914810f6f01c79e5da88794972fa4f1439c4Ying Wang * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 156099914810f6f01c79e5da88794972fa4f1439c4Ying Wang * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 166099914810f6f01c79e5da88794972fa4f1439c4Ying Wang * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 176099914810f6f01c79e5da88794972fa4f1439c4Ying Wang * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 186099914810f6f01c79e5da88794972fa4f1439c4Ying Wang * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 196099914810f6f01c79e5da88794972fa4f1439c4Ying Wang * THE SOFTWARE. 206099914810f6f01c79e5da88794972fa4f1439c4Ying Wang * 216099914810f6f01c79e5da88794972fa4f1439c4Ying Wang *===-----------------------------------------------------------------------=== 226099914810f6f01c79e5da88794972fa4f1439c4Ying Wang */ 236099914810f6f01c79e5da88794972fa4f1439c4Ying Wang 246099914810f6f01c79e5da88794972fa4f1439c4Ying Wang#if !defined __X86INTRIN_H && !defined __IMMINTRIN_H 256099914810f6f01c79e5da88794972fa4f1439c4Ying Wang#error "Never use <bmi2intrin.h> directly; include <x86intrin.h> instead." 266099914810f6f01c79e5da88794972fa4f1439c4Ying Wang#endif 276099914810f6f01c79e5da88794972fa4f1439c4Ying Wang 286099914810f6f01c79e5da88794972fa4f1439c4Ying Wang#ifndef __BMI2__ 296099914810f6f01c79e5da88794972fa4f1439c4Ying Wang# error "BMI2 instruction set not enabled" 306099914810f6f01c79e5da88794972fa4f1439c4Ying Wang#endif /* __BMI2__ */ 316099914810f6f01c79e5da88794972fa4f1439c4Ying Wang 326099914810f6f01c79e5da88794972fa4f1439c4Ying Wang#ifndef __BMI2INTRIN_H 336099914810f6f01c79e5da88794972fa4f1439c4Ying Wang#define __BMI2INTRIN_H 346099914810f6f01c79e5da88794972fa4f1439c4Ying Wang 356099914810f6f01c79e5da88794972fa4f1439c4Ying Wangstatic __inline__ unsigned int __attribute__((__always_inline__, __nodebug__)) 366099914810f6f01c79e5da88794972fa4f1439c4Ying Wang_bzhi_u32(unsigned int __X, unsigned int __Y) 376099914810f6f01c79e5da88794972fa4f1439c4Ying Wang{ 386099914810f6f01c79e5da88794972fa4f1439c4Ying Wang return __builtin_ia32_bzhi_si(__X, __Y); 396099914810f6f01c79e5da88794972fa4f1439c4Ying Wang} 406099914810f6f01c79e5da88794972fa4f1439c4Ying Wang 416099914810f6f01c79e5da88794972fa4f1439c4Ying Wangstatic __inline__ unsigned int __attribute__((__always_inline__, __nodebug__)) 426099914810f6f01c79e5da88794972fa4f1439c4Ying Wang_pdep_u32(unsigned int __X, unsigned int __Y) 436099914810f6f01c79e5da88794972fa4f1439c4Ying Wang{ 446099914810f6f01c79e5da88794972fa4f1439c4Ying Wang return __builtin_ia32_pdep_si(__X, __Y); 456099914810f6f01c79e5da88794972fa4f1439c4Ying Wang} 466099914810f6f01c79e5da88794972fa4f1439c4Ying Wang 476099914810f6f01c79e5da88794972fa4f1439c4Ying Wangstatic __inline__ unsigned int __attribute__((__always_inline__, __nodebug__)) 486099914810f6f01c79e5da88794972fa4f1439c4Ying Wang_pext_u32(unsigned int __X, unsigned int __Y) 496099914810f6f01c79e5da88794972fa4f1439c4Ying Wang{ 506099914810f6f01c79e5da88794972fa4f1439c4Ying Wang return __builtin_ia32_pext_si(__X, __Y); 516099914810f6f01c79e5da88794972fa4f1439c4Ying Wang} 526099914810f6f01c79e5da88794972fa4f1439c4Ying Wang 536099914810f6f01c79e5da88794972fa4f1439c4Ying Wang#ifdef __x86_64__ 546099914810f6f01c79e5da88794972fa4f1439c4Ying Wang 556099914810f6f01c79e5da88794972fa4f1439c4Ying Wangstatic __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__)) 566099914810f6f01c79e5da88794972fa4f1439c4Ying Wang_bzhi_u64(unsigned long long __X, unsigned long long __Y) 576099914810f6f01c79e5da88794972fa4f1439c4Ying Wang{ 586099914810f6f01c79e5da88794972fa4f1439c4Ying Wang return __builtin_ia32_bzhi_di(__X, __Y); 596099914810f6f01c79e5da88794972fa4f1439c4Ying Wang} 606099914810f6f01c79e5da88794972fa4f1439c4Ying Wang 616099914810f6f01c79e5da88794972fa4f1439c4Ying Wangstatic __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__)) 626099914810f6f01c79e5da88794972fa4f1439c4Ying Wang_pdep_u64(unsigned long long __X, unsigned long long __Y) 636099914810f6f01c79e5da88794972fa4f1439c4Ying Wang{ 646099914810f6f01c79e5da88794972fa4f1439c4Ying Wang return __builtin_ia32_pdep_di(__X, __Y); 656099914810f6f01c79e5da88794972fa4f1439c4Ying Wang} 666099914810f6f01c79e5da88794972fa4f1439c4Ying Wang 676099914810f6f01c79e5da88794972fa4f1439c4Ying Wangstatic __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__)) 686099914810f6f01c79e5da88794972fa4f1439c4Ying Wang_pext_u64(unsigned long long __X, unsigned long long __Y) 696099914810f6f01c79e5da88794972fa4f1439c4Ying Wang{ 706099914810f6f01c79e5da88794972fa4f1439c4Ying Wang return __builtin_ia32_pext_di(__X, __Y); 716099914810f6f01c79e5da88794972fa4f1439c4Ying Wang} 726099914810f6f01c79e5da88794972fa4f1439c4Ying Wang 73c6ee7dfdcb47cdfbdb2e3cb056d590f87ce3bcb1Stephen Hinesstatic __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__)) 74c6ee7dfdcb47cdfbdb2e3cb056d590f87ce3bcb1Stephen Hines_mulx_u64 (unsigned long long __X, unsigned long long __Y, 75c6ee7dfdcb47cdfbdb2e3cb056d590f87ce3bcb1Stephen Hines unsigned long long *__P) 76c6ee7dfdcb47cdfbdb2e3cb056d590f87ce3bcb1Stephen Hines{ 77c6ee7dfdcb47cdfbdb2e3cb056d590f87ce3bcb1Stephen Hines unsigned __int128 __res = (unsigned __int128) __X * __Y; 78c6ee7dfdcb47cdfbdb2e3cb056d590f87ce3bcb1Stephen Hines *__P = (unsigned long long) (__res >> 64); 79c6ee7dfdcb47cdfbdb2e3cb056d590f87ce3bcb1Stephen Hines return (unsigned long long) __res; 80c6ee7dfdcb47cdfbdb2e3cb056d590f87ce3bcb1Stephen Hines} 81c6ee7dfdcb47cdfbdb2e3cb056d590f87ce3bcb1Stephen Hines 82c6ee7dfdcb47cdfbdb2e3cb056d590f87ce3bcb1Stephen Hines#else /* !__x86_64__ */ 83c6ee7dfdcb47cdfbdb2e3cb056d590f87ce3bcb1Stephen Hines 84c6ee7dfdcb47cdfbdb2e3cb056d590f87ce3bcb1Stephen Hinesstatic __inline__ unsigned int __attribute__((__always_inline__, __nodebug__)) 85c6ee7dfdcb47cdfbdb2e3cb056d590f87ce3bcb1Stephen Hines_mulx_u32 (unsigned int __X, unsigned int __Y, unsigned int *__P) 86c6ee7dfdcb47cdfbdb2e3cb056d590f87ce3bcb1Stephen Hines{ 87c6ee7dfdcb47cdfbdb2e3cb056d590f87ce3bcb1Stephen Hines unsigned long long __res = (unsigned long long) __X * __Y; 88c6ee7dfdcb47cdfbdb2e3cb056d590f87ce3bcb1Stephen Hines *__P = (unsigned int) (__res >> 32); 89c6ee7dfdcb47cdfbdb2e3cb056d590f87ce3bcb1Stephen Hines return (unsigned int) __res; 90c6ee7dfdcb47cdfbdb2e3cb056d590f87ce3bcb1Stephen Hines} 91c6ee7dfdcb47cdfbdb2e3cb056d590f87ce3bcb1Stephen Hines 926099914810f6f01c79e5da88794972fa4f1439c4Ying Wang#endif /* !__x86_64__ */ 936099914810f6f01c79e5da88794972fa4f1439c4Ying Wang 946099914810f6f01c79e5da88794972fa4f1439c4Ying Wang#endif /* __BMI2INTRIN_H */ 95