1/*===---- immintrin.h - Intel intrinsics -----------------------------------===
2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy
4 * of this software and associated documentation files (the "Software"), to deal
5 * in the Software without restriction, including without limitation the rights
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *
21 *===-----------------------------------------------------------------------===
22 */
23
24#ifndef __IMMINTRIN_H
25#define __IMMINTRIN_H
26
27#if !defined(_MSC_VER) || __has_feature(modules) || defined(__MMX__)
28#include <mmintrin.h>
29#endif
30
31#if !defined(_MSC_VER) || __has_feature(modules) || defined(__SSE__)
32#include <xmmintrin.h>
33#endif
34
35#if !defined(_MSC_VER) || __has_feature(modules) || defined(__SSE2__)
36#include <emmintrin.h>
37#endif
38
39#if !defined(_MSC_VER) || __has_feature(modules) || defined(__SSE3__)
40#include <pmmintrin.h>
41#endif
42
43#if !defined(_MSC_VER) || __has_feature(modules) || defined(__SSSE3__)
44#include <tmmintrin.h>
45#endif
46
47#if !defined(_MSC_VER) || __has_feature(modules) || \
48    (defined(__SSE4_2__) || defined(__SSE4_1__))
49#include <smmintrin.h>
50#endif
51
52#if !defined(_MSC_VER) || __has_feature(modules) || \
53    (defined(__AES__) || defined(__PCLMUL__))
54#include <wmmintrin.h>
55#endif
56
57#if !defined(_MSC_VER) || __has_feature(modules) || defined(__CLFLUSHOPT__)
58#include <clflushoptintrin.h>
59#endif
60
61#if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX__)
62#include <avxintrin.h>
63#endif
64
65#if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX2__)
66#include <avx2intrin.h>
67
68/* The 256-bit versions of functions in f16cintrin.h.
69   Intel documents these as being in immintrin.h, and
70   they depend on typedefs from avxintrin.h. */
71
72#define _mm256_cvtps_ph(a, imm) __extension__ ({ \
73 (__m128i)__builtin_ia32_vcvtps2ph256((__v8sf)(__m256)(a), (imm)); })
74
75static __inline __m256 __attribute__((__always_inline__, __nodebug__, __target__("f16c")))
76_mm256_cvtph_ps(__m128i __a)
77{
78  return (__m256)__builtin_ia32_vcvtph2ps256((__v8hi)__a);
79}
80#endif /* __AVX2__ */
81
82#if !defined(_MSC_VER) || __has_feature(modules) || defined(__BMI__)
83#include <bmiintrin.h>
84#endif
85
86#if !defined(_MSC_VER) || __has_feature(modules) || defined(__BMI2__)
87#include <bmi2intrin.h>
88#endif
89
90#if !defined(_MSC_VER) || __has_feature(modules) || defined(__LZCNT__)
91#include <lzcntintrin.h>
92#endif
93
94#if !defined(_MSC_VER) || __has_feature(modules) || defined(__FMA__)
95#include <fmaintrin.h>
96#endif
97
98#if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512F__)
99#include <avx512fintrin.h>
100#endif
101
102#if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512VL__)
103#include <avx512vlintrin.h>
104#endif
105
106#if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512BW__)
107#include <avx512bwintrin.h>
108#endif
109
110#if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512CD__)
111#include <avx512cdintrin.h>
112#endif
113
114#if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512DQ__)
115#include <avx512dqintrin.h>
116#endif
117
118#if !defined(_MSC_VER) || __has_feature(modules) || \
119    (defined(__AVX512VL__) && defined(__AVX512BW__))
120#include <avx512vlbwintrin.h>
121#endif
122
123#if !defined(_MSC_VER) || __has_feature(modules) || \
124    (defined(__AVX512VL__) && defined(__AVX512CD__))
125#include <avx512vlcdintrin.h>
126#endif
127
128#if !defined(_MSC_VER) || __has_feature(modules) || \
129    (defined(__AVX512VL__) && defined(__AVX512DQ__))
130#include <avx512vldqintrin.h>
131#endif
132
133#if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512ER__)
134#include <avx512erintrin.h>
135#endif
136
137#if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512IFMA__)
138#include <avx512ifmaintrin.h>
139#endif
140
141#if !defined(_MSC_VER) || __has_feature(modules) || \
142    (defined(__AVX512IFMA__) && defined(__AVX512VL__))
143#include <avx512ifmavlintrin.h>
144#endif
145
146#if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512VBMI__)
147#include <avx512vbmiintrin.h>
148#endif
149
150#if !defined(_MSC_VER) || __has_feature(modules) || \
151    (defined(__AVX512VBMI__) && defined(__AVX512VL__))
152#include <avx512vbmivlintrin.h>
153#endif
154
155#if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512PF__)
156#include <avx512pfintrin.h>
157#endif
158
159#if !defined(_MSC_VER) || __has_feature(modules) || defined(__PKU__)
160#include <pkuintrin.h>
161#endif
162
163#if !defined(_MSC_VER) || __has_feature(modules) || defined(__RDRND__)
164static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
165_rdrand16_step(unsigned short *__p)
166{
167  return __builtin_ia32_rdrand16_step(__p);
168}
169
170static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
171_rdrand32_step(unsigned int *__p)
172{
173  return __builtin_ia32_rdrand32_step(__p);
174}
175
176/* __bit_scan_forward */
177static __inline__ int __attribute__((__always_inline__, __nodebug__))
178_bit_scan_forward(int __A) {
179  return __builtin_ctz(__A);
180}
181
182/* __bit_scan_reverse */
183static __inline__ int __attribute__((__always_inline__, __nodebug__))
184_bit_scan_reverse(int __A) {
185  return 31 - __builtin_clz(__A);
186}
187
188#ifdef __x86_64__
189static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
190_rdrand64_step(unsigned long long *__p)
191{
192  return __builtin_ia32_rdrand64_step(__p);
193}
194#endif
195#endif /* __RDRND__ */
196
197#if !defined(_MSC_VER) || __has_feature(modules) || defined(__FSGSBASE__)
198#ifdef __x86_64__
199static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
200_readfsbase_u32(void)
201{
202  return __builtin_ia32_rdfsbase32();
203}
204
205static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
206_readfsbase_u64(void)
207{
208  return __builtin_ia32_rdfsbase64();
209}
210
211static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
212_readgsbase_u32(void)
213{
214  return __builtin_ia32_rdgsbase32();
215}
216
217static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
218_readgsbase_u64(void)
219{
220  return __builtin_ia32_rdgsbase64();
221}
222
223static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
224_writefsbase_u32(unsigned int __V)
225{
226  return __builtin_ia32_wrfsbase32(__V);
227}
228
229static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
230_writefsbase_u64(unsigned long long __V)
231{
232  return __builtin_ia32_wrfsbase64(__V);
233}
234
235static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
236_writegsbase_u32(unsigned int __V)
237{
238  return __builtin_ia32_wrgsbase32(__V);
239}
240
241static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
242_writegsbase_u64(unsigned long long __V)
243{
244  return __builtin_ia32_wrgsbase64(__V);
245}
246
247#endif
248#endif /* __FSGSBASE__ */
249
250#if !defined(_MSC_VER) || __has_feature(modules) || defined(__RTM__)
251#include <rtmintrin.h>
252#include <xtestintrin.h>
253#endif
254
255#if !defined(_MSC_VER) || __has_feature(modules) || defined(__SHA__)
256#include <shaintrin.h>
257#endif
258
259#if !defined(_MSC_VER) || __has_feature(modules) || defined(__FXSR__)
260#include <fxsrintrin.h>
261#endif
262
263#if !defined(_MSC_VER) || __has_feature(modules) || defined(__XSAVE__)
264#include <xsaveintrin.h>
265#endif
266
267#if !defined(_MSC_VER) || __has_feature(modules) || defined(__XSAVEOPT__)
268#include <xsaveoptintrin.h>
269#endif
270
271#if !defined(_MSC_VER) || __has_feature(modules) || defined(__XSAVEC__)
272#include <xsavecintrin.h>
273#endif
274
275#if !defined(_MSC_VER) || __has_feature(modules) || defined(__XSAVES__)
276#include <xsavesintrin.h>
277#endif
278
279/* Some intrinsics inside adxintrin.h are available only on processors with ADX,
280 * whereas others are also available at all times. */
281#include <adxintrin.h>
282
283#endif /* __IMMINTRIN_H */
284