11a37d0a8f0f7110046e499bbe1117fec91f31f7eAnders Carlsson/*===---- smmintrin.h - SSE4 intrinsics ------------------------------------=== 21a37d0a8f0f7110046e499bbe1117fec91f31f7eAnders Carlsson * 31a37d0a8f0f7110046e499bbe1117fec91f31f7eAnders Carlsson * Permission is hereby granted, free of charge, to any person obtaining a copy 41a37d0a8f0f7110046e499bbe1117fec91f31f7eAnders Carlsson * of this software and associated documentation files (the "Software"), to deal 51a37d0a8f0f7110046e499bbe1117fec91f31f7eAnders Carlsson * in the Software without restriction, including without limitation the rights 61a37d0a8f0f7110046e499bbe1117fec91f31f7eAnders Carlsson * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 71a37d0a8f0f7110046e499bbe1117fec91f31f7eAnders Carlsson * copies of the Software, and to permit persons to whom the Software is 81a37d0a8f0f7110046e499bbe1117fec91f31f7eAnders Carlsson * furnished to do so, subject to the following conditions: 91a37d0a8f0f7110046e499bbe1117fec91f31f7eAnders Carlsson * 101a37d0a8f0f7110046e499bbe1117fec91f31f7eAnders Carlsson * The above copyright notice and this permission notice shall be included in 111a37d0a8f0f7110046e499bbe1117fec91f31f7eAnders Carlsson * all copies or substantial portions of the Software. 121a37d0a8f0f7110046e499bbe1117fec91f31f7eAnders Carlsson * 131a37d0a8f0f7110046e499bbe1117fec91f31f7eAnders Carlsson * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 141a37d0a8f0f7110046e499bbe1117fec91f31f7eAnders Carlsson * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 151a37d0a8f0f7110046e499bbe1117fec91f31f7eAnders Carlsson * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 161a37d0a8f0f7110046e499bbe1117fec91f31f7eAnders Carlsson * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 171a37d0a8f0f7110046e499bbe1117fec91f31f7eAnders Carlsson * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 181a37d0a8f0f7110046e499bbe1117fec91f31f7eAnders Carlsson * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 191a37d0a8f0f7110046e499bbe1117fec91f31f7eAnders Carlsson * THE SOFTWARE. 201a37d0a8f0f7110046e499bbe1117fec91f31f7eAnders Carlsson * 211a37d0a8f0f7110046e499bbe1117fec91f31f7eAnders Carlsson *===-----------------------------------------------------------------------=== 221a37d0a8f0f7110046e499bbe1117fec91f31f7eAnders Carlsson */ 23b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher 24b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#ifndef _SMMINTRIN_H 25b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#define _SMMINTRIN_H 26b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher 27b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#ifndef __SSE4_1__ 28b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#error "SSE4.1 instruction set not enabled" 29b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#else 30b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher 31b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#include <tmmintrin.h> 32b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher 33b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher/* SSE4 Rounding macros. */ 34b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#define _MM_FROUND_TO_NEAREST_INT 0x00 35b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#define _MM_FROUND_TO_NEG_INF 0x01 36b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#define _MM_FROUND_TO_POS_INF 0x02 37b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#define _MM_FROUND_TO_ZERO 0x03 38b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#define _MM_FROUND_CUR_DIRECTION 0x04 39b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher 40b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#define _MM_FROUND_RAISE_EXC 0x00 41b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#define _MM_FROUND_NO_EXC 0x08 42b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher 43b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#define _MM_FROUND_NINT (_MM_FROUND_RAISE_EXC | _MM_FROUND_TO_NEAREST_INT) 44b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#define _MM_FROUND_FLOOR (_MM_FROUND_RAISE_EXC | _MM_FROUND_TO_NEG_INF) 45b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#define _MM_FROUND_CEIL (_MM_FROUND_RAISE_EXC | _MM_FROUND_TO_POS_INF) 46b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#define _MM_FROUND_TRUNC (_MM_FROUND_RAISE_EXC | _MM_FROUND_TO_ZERO) 47b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#define _MM_FROUND_RINT (_MM_FROUND_RAISE_EXC | _MM_FROUND_CUR_DIRECTION) 489ce2116005d417f7a36fa99bba7e499a58650a9aEric Christopher#define _MM_FROUND_NEARBYINT (_MM_FROUND_NO_EXC | _MM_FROUND_CUR_DIRECTION) 49b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher 50b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#define _mm_ceil_ps(X) _mm_round_ps((X), _MM_FROUND_CEIL) 51b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#define _mm_ceil_pd(X) _mm_round_pd((X), _MM_FROUND_CEIL) 52b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#define _mm_ceil_ss(X, Y) _mm_round_ss((X), (Y), _MM_FROUND_CEIL) 53b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#define _mm_ceil_sd(X, Y) _mm_round_sd((X), (Y), _MM_FROUND_CEIL) 54b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher 55b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#define _mm_floor_ps(X) _mm_round_ps((X), _MM_FROUND_FLOOR) 56b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#define _mm_floor_pd(X) _mm_round_pd((X), _MM_FROUND_FLOOR) 57b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#define _mm_floor_ss(X, Y) _mm_round_ss((X), (Y), _MM_FROUND_FLOOR) 58b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#define _mm_floor_sd(X, Y) _mm_round_sd((X), (Y), _MM_FROUND_FLOOR) 59b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher 605aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper#define _mm_round_ps(X, M) __extension__ ({ \ 615aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper __m128 __X = (X); \ 625aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper (__m128) __builtin_ia32_roundps((__v4sf)__X, (M)); }) 635aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper 645aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper#define _mm_round_ss(X, Y, M) __extension__ ({ \ 655aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper __m128 __X = (X); \ 665aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper __m128 __Y = (Y); \ 675aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper (__m128) __builtin_ia32_roundss((__v4sf)__X, (__v4sf)__Y, (M)); }) 685aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper 695aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper#define _mm_round_pd(X, M) __extension__ ({ \ 705aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper __m128d __X = (X); \ 715aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper (__m128d) __builtin_ia32_roundpd((__v2df)__X, (M)); }) 725aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper 735aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper#define _mm_round_sd(X, Y, M) __extension__ ({ \ 745aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper __m128d __X = (X); \ 755aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper __m128d __Y = (Y); \ 765aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper (__m128d) __builtin_ia32_roundsd((__v2df)__X, (__v2df)__Y, (M)); }) 77b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher 78b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher/* SSE4 Packed Blending Intrinsics. */ 79347208968c303a9c11fe29012f6dc49680465182Eli Friedman#define _mm_blend_pd(V1, V2, M) __extension__ ({ \ 80347208968c303a9c11fe29012f6dc49680465182Eli Friedman __m128d __V1 = (V1); \ 81347208968c303a9c11fe29012f6dc49680465182Eli Friedman __m128d __V2 = (V2); \ 826bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines (__m128d)__builtin_shufflevector((__v2df)__V1, (__v2df)__V2, \ 836bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines (((M) & 0x01) ? 2 : 0), \ 846bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines (((M) & 0x02) ? 3 : 1)); }) 85b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher 86347208968c303a9c11fe29012f6dc49680465182Eli Friedman#define _mm_blend_ps(V1, V2, M) __extension__ ({ \ 87347208968c303a9c11fe29012f6dc49680465182Eli Friedman __m128 __V1 = (V1); \ 88347208968c303a9c11fe29012f6dc49680465182Eli Friedman __m128 __V2 = (V2); \ 896bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines (__m128)__builtin_shufflevector((__v4sf)__V1, (__v4sf)__V2, \ 906bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines (((M) & 0x01) ? 4 : 0), \ 916bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines (((M) & 0x02) ? 5 : 1), \ 926bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines (((M) & 0x04) ? 6 : 2), \ 936bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines (((M) & 0x08) ? 7 : 3)); }) 94b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher 951bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128d __attribute__((__always_inline__, __nodebug__)) 96b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher_mm_blendv_pd (__m128d __V1, __m128d __V2, __m128d __M) 97b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher{ 98b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher return (__m128d) __builtin_ia32_blendvpd ((__v2df)__V1, (__v2df)__V2, 99b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher (__v2df)__M); 100b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher} 101b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher 1021bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128 __attribute__((__always_inline__, __nodebug__)) 103b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher_mm_blendv_ps (__m128 __V1, __m128 __V2, __m128 __M) 104b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher{ 105b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher return (__m128) __builtin_ia32_blendvps ((__v4sf)__V1, (__v4sf)__V2, 106b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher (__v4sf)__M); 107b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher} 108b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher 1091bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 110b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher_mm_blendv_epi8 (__m128i __V1, __m128i __V2, __m128i __M) 111b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher{ 112b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher return (__m128i) __builtin_ia32_pblendvb128 ((__v16qi)__V1, (__v16qi)__V2, 113b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher (__v16qi)__M); 114b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher} 115b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher 116347208968c303a9c11fe29012f6dc49680465182Eli Friedman#define _mm_blend_epi16(V1, V2, M) __extension__ ({ \ 117347208968c303a9c11fe29012f6dc49680465182Eli Friedman __m128i __V1 = (V1); \ 118347208968c303a9c11fe29012f6dc49680465182Eli Friedman __m128i __V2 = (V2); \ 1196bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines (__m128i)__builtin_shufflevector((__v8hi)__V1, (__v8hi)__V2, \ 1206bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines (((M) & 0x01) ? 8 : 0), \ 1216bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines (((M) & 0x02) ? 9 : 1), \ 1226bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines (((M) & 0x04) ? 10 : 2), \ 1236bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines (((M) & 0x08) ? 11 : 3), \ 1246bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines (((M) & 0x10) ? 12 : 4), \ 1256bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines (((M) & 0x20) ? 13 : 5), \ 1266bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines (((M) & 0x40) ? 14 : 6), \ 1276bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines (((M) & 0x80) ? 15 : 7)); }) 128b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher 129c82ac95dbd68cbfa13bf8a93f7230fbe252931f7Eric Christopher/* SSE4 Dword Multiply Instructions. */ 1301bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 131c82ac95dbd68cbfa13bf8a93f7230fbe252931f7Eric Christopher_mm_mullo_epi32 (__m128i __V1, __m128i __V2) 132c82ac95dbd68cbfa13bf8a93f7230fbe252931f7Eric Christopher{ 133054c27674efe41608eb5546fe16e43f4b3777407Eric Christopher return (__m128i) ((__v4si)__V1 * (__v4si)__V2); 134c82ac95dbd68cbfa13bf8a93f7230fbe252931f7Eric Christopher} 135c82ac95dbd68cbfa13bf8a93f7230fbe252931f7Eric Christopher 1361bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 137c82ac95dbd68cbfa13bf8a93f7230fbe252931f7Eric Christopher_mm_mul_epi32 (__m128i __V1, __m128i __V2) 138c82ac95dbd68cbfa13bf8a93f7230fbe252931f7Eric Christopher{ 139c82ac95dbd68cbfa13bf8a93f7230fbe252931f7Eric Christopher return (__m128i) __builtin_ia32_pmuldq128 ((__v4si)__V1, (__v4si)__V2); 140c82ac95dbd68cbfa13bf8a93f7230fbe252931f7Eric Christopher} 141c82ac95dbd68cbfa13bf8a93f7230fbe252931f7Eric Christopher 142c82ac95dbd68cbfa13bf8a93f7230fbe252931f7Eric Christopher/* SSE4 Floating Point Dot Product Instructions. */ 1435aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper#define _mm_dp_ps(X, Y, M) __extension__ ({ \ 1445aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper __m128 __X = (X); \ 1455aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper __m128 __Y = (Y); \ 1465aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper (__m128) __builtin_ia32_dpps((__v4sf)__X, (__v4sf)__Y, (M)); }) 1475aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper 1485aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper#define _mm_dp_pd(X, Y, M) __extension__ ({\ 1495aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper __m128d __X = (X); \ 1505aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper __m128d __Y = (Y); \ 1515aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper (__m128d) __builtin_ia32_dppd((__v2df)__X, (__v2df)__Y, (M)); }) 152c82ac95dbd68cbfa13bf8a93f7230fbe252931f7Eric Christopher 15303568487447d3d8240c36727120a6dd9c527ec5eEric Christopher/* SSE4 Streaming Load Hint Instruction. */ 1541bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 15503568487447d3d8240c36727120a6dd9c527ec5eEric Christopher_mm_stream_load_si128 (__m128i *__V) 15603568487447d3d8240c36727120a6dd9c527ec5eEric Christopher{ 15703568487447d3d8240c36727120a6dd9c527ec5eEric Christopher return (__m128i) __builtin_ia32_movntdqa ((__v2di *) __V); 15803568487447d3d8240c36727120a6dd9c527ec5eEric Christopher} 15903568487447d3d8240c36727120a6dd9c527ec5eEric Christopher 1603afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher/* SSE4 Packed Integer Min/Max Instructions. */ 1611bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 1623afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher_mm_min_epi8 (__m128i __V1, __m128i __V2) 1633afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher{ 1643afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher return (__m128i) __builtin_ia32_pminsb128 ((__v16qi) __V1, (__v16qi) __V2); 1653afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher} 1663afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher 1671bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 1683afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher_mm_max_epi8 (__m128i __V1, __m128i __V2) 1693afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher{ 1703afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher return (__m128i) __builtin_ia32_pmaxsb128 ((__v16qi) __V1, (__v16qi) __V2); 1713afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher} 1723afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher 1731bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 1743afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher_mm_min_epu16 (__m128i __V1, __m128i __V2) 1753afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher{ 1763afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher return (__m128i) __builtin_ia32_pminuw128 ((__v8hi) __V1, (__v8hi) __V2); 1773afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher} 1783afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher 1791bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 1803afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher_mm_max_epu16 (__m128i __V1, __m128i __V2) 1813afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher{ 1823afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher return (__m128i) __builtin_ia32_pmaxuw128 ((__v8hi) __V1, (__v8hi) __V2); 1833afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher} 1843afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher 1851bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 1863afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher_mm_min_epi32 (__m128i __V1, __m128i __V2) 1873afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher{ 1883afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher return (__m128i) __builtin_ia32_pminsd128 ((__v4si) __V1, (__v4si) __V2); 1893afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher} 1903afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher 1911bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 1923afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher_mm_max_epi32 (__m128i __V1, __m128i __V2) 1933afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher{ 1943afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher return (__m128i) __builtin_ia32_pmaxsd128 ((__v4si) __V1, (__v4si) __V2); 1953afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher} 1963afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher 1971bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 1983afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher_mm_min_epu32 (__m128i __V1, __m128i __V2) 1993afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher{ 2003afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher return (__m128i) __builtin_ia32_pminud128((__v4si) __V1, (__v4si) __V2); 2013afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher} 2023afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher 2031bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 2043afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher_mm_max_epu32 (__m128i __V1, __m128i __V2) 2053afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher{ 2063afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher return (__m128i) __builtin_ia32_pmaxud128((__v4si) __V1, (__v4si) __V2); 2073afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher} 2083afda6037d0c96a1868fe9a5ab51b6cfe7fade1cEric Christopher 2099a2a69f60898f771d44af0e40e7b4cf37b565d21Eric Christopher/* SSE4 Insertion and Extraction from XMM Register Instructions. */ 2109a2a69f60898f771d44af0e40e7b4cf37b565d21Eric Christopher#define _mm_insert_ps(X, Y, N) __builtin_ia32_insertps128((X), (Y), (N)) 2119a2a69f60898f771d44af0e40e7b4cf37b565d21Eric Christopher#define _mm_extract_ps(X, N) (__extension__ \ 2124f918aed75d4927e88365541c7200f0b5fe5014bDavid Blaikie ({ union { int __i; float __f; } __t; \ 21379e5ab7a537987348a9ba01424d8bbe7080eac57Daniel Dunbar __v4sf __a = (__v4sf)(X); \ 2148484375b0fc442c704b3ec06e2e795d88591606fManman Ren __t.__f = __a[(N) & 3]; \ 2154f918aed75d4927e88365541c7200f0b5fe5014bDavid Blaikie __t.__i;})) 2169a2a69f60898f771d44af0e40e7b4cf37b565d21Eric Christopher 2179a2a69f60898f771d44af0e40e7b4cf37b565d21Eric Christopher/* Miscellaneous insert and extract macros. */ 2189a2a69f60898f771d44af0e40e7b4cf37b565d21Eric Christopher/* Extract a single-precision float from X at index N into D. */ 21979e5ab7a537987348a9ba01424d8bbe7080eac57Daniel Dunbar#define _MM_EXTRACT_FLOAT(D, X, N) (__extension__ ({ __v4sf __a = (__v4sf)(X); \ 2209a2a69f60898f771d44af0e40e7b4cf37b565d21Eric Christopher (D) = __a[N]; })) 2219a2a69f60898f771d44af0e40e7b4cf37b565d21Eric Christopher 2229a2a69f60898f771d44af0e40e7b4cf37b565d21Eric Christopher/* Or together 2 sets of indexes (X and Y) with the zeroing bits (Z) to create 2239a2a69f60898f771d44af0e40e7b4cf37b565d21Eric Christopher an index suitable for _mm_insert_ps. */ 2249a2a69f60898f771d44af0e40e7b4cf37b565d21Eric Christopher#define _MM_MK_INSERTPS_NDX(X, Y, Z) (((X) << 6) | ((Y) << 4) | (Z)) 2259a2a69f60898f771d44af0e40e7b4cf37b565d21Eric Christopher 2269a2a69f60898f771d44af0e40e7b4cf37b565d21Eric Christopher/* Extract a float from X at index N into the first index of the return. */ 2279a2a69f60898f771d44af0e40e7b4cf37b565d21Eric Christopher#define _MM_PICK_OUT_PS(X, N) _mm_insert_ps (_mm_setzero_ps(), (X), \ 2289a2a69f60898f771d44af0e40e7b4cf37b565d21Eric Christopher _MM_MK_INSERTPS_NDX((N), 0, 0x0e)) 2296fa43e3aaf98afc3b337968429523e80f6e08d91Eric Christopher 2306fa43e3aaf98afc3b337968429523e80f6e08d91Eric Christopher/* Insert int into packed integer array at index. */ 23179e5ab7a537987348a9ba01424d8bbe7080eac57Daniel Dunbar#define _mm_insert_epi8(X, I, N) (__extension__ ({ __v16qi __a = (__v16qi)(X); \ 2328484375b0fc442c704b3ec06e2e795d88591606fManman Ren __a[(N) & 15] = (I); \ 2336fa43e3aaf98afc3b337968429523e80f6e08d91Eric Christopher __a;})) 23479e5ab7a537987348a9ba01424d8bbe7080eac57Daniel Dunbar#define _mm_insert_epi32(X, I, N) (__extension__ ({ __v4si __a = (__v4si)(X); \ 2358484375b0fc442c704b3ec06e2e795d88591606fManman Ren __a[(N) & 3] = (I); \ 2366fa43e3aaf98afc3b337968429523e80f6e08d91Eric Christopher __a;})) 2376fa43e3aaf98afc3b337968429523e80f6e08d91Eric Christopher#ifdef __x86_64__ 23879e5ab7a537987348a9ba01424d8bbe7080eac57Daniel Dunbar#define _mm_insert_epi64(X, I, N) (__extension__ ({ __v2di __a = (__v2di)(X); \ 2398484375b0fc442c704b3ec06e2e795d88591606fManman Ren __a[(N) & 1] = (I); \ 2406fa43e3aaf98afc3b337968429523e80f6e08d91Eric Christopher __a;})) 2416fa43e3aaf98afc3b337968429523e80f6e08d91Eric Christopher#endif /* __x86_64__ */ 2429a2a69f60898f771d44af0e40e7b4cf37b565d21Eric Christopher 243d6b84b9455202c8a93a306c02770cdfecf13fda0Chris Lattner/* Extract int from packed integer array at index. This returns the element 244d6b84b9455202c8a93a306c02770cdfecf13fda0Chris Lattner * as a zero extended value, so it is unsigned. 245d6b84b9455202c8a93a306c02770cdfecf13fda0Chris Lattner */ 24679e5ab7a537987348a9ba01424d8bbe7080eac57Daniel Dunbar#define _mm_extract_epi8(X, N) (__extension__ ({ __v16qi __a = (__v16qi)(X); \ 247860c518c99f23e9c78c941627e91d0fa735a27c6Eli Friedman (int)(unsigned char) \ 2488484375b0fc442c704b3ec06e2e795d88591606fManman Ren __a[(N) & 15];})) 24979e5ab7a537987348a9ba01424d8bbe7080eac57Daniel Dunbar#define _mm_extract_epi32(X, N) (__extension__ ({ __v4si __a = (__v4si)(X); \ 2508484375b0fc442c704b3ec06e2e795d88591606fManman Ren __a[(N) & 3];})) 251ff8aeaa9827188f9fc360cb4e0a6c8b6331edc8aEric Christopher#ifdef __x86_64__ 25279e5ab7a537987348a9ba01424d8bbe7080eac57Daniel Dunbar#define _mm_extract_epi64(X, N) (__extension__ ({ __v2di __a = (__v2di)(X); \ 2538484375b0fc442c704b3ec06e2e795d88591606fManman Ren __a[(N) & 1];})) 254ff8aeaa9827188f9fc360cb4e0a6c8b6331edc8aEric Christopher#endif /* __x86_64 */ 255ff8aeaa9827188f9fc360cb4e0a6c8b6331edc8aEric Christopher 2560c2b4f40ac17156a424d4faa20ba5d862ecc8416Eric Christopher/* SSE4 128-bit Packed Integer Comparisons. */ 2571bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ int __attribute__((__always_inline__, __nodebug__)) 2580c2b4f40ac17156a424d4faa20ba5d862ecc8416Eric Christopher_mm_testz_si128(__m128i __M, __m128i __V) 2590c2b4f40ac17156a424d4faa20ba5d862ecc8416Eric Christopher{ 2600c2b4f40ac17156a424d4faa20ba5d862ecc8416Eric Christopher return __builtin_ia32_ptestz128((__v2di)__M, (__v2di)__V); 2610c2b4f40ac17156a424d4faa20ba5d862ecc8416Eric Christopher} 2620c2b4f40ac17156a424d4faa20ba5d862ecc8416Eric Christopher 2631bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ int __attribute__((__always_inline__, __nodebug__)) 2640c2b4f40ac17156a424d4faa20ba5d862ecc8416Eric Christopher_mm_testc_si128(__m128i __M, __m128i __V) 2650c2b4f40ac17156a424d4faa20ba5d862ecc8416Eric Christopher{ 2660c2b4f40ac17156a424d4faa20ba5d862ecc8416Eric Christopher return __builtin_ia32_ptestc128((__v2di)__M, (__v2di)__V); 2670c2b4f40ac17156a424d4faa20ba5d862ecc8416Eric Christopher} 2680c2b4f40ac17156a424d4faa20ba5d862ecc8416Eric Christopher 2691bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ int __attribute__((__always_inline__, __nodebug__)) 2700c2b4f40ac17156a424d4faa20ba5d862ecc8416Eric Christopher_mm_testnzc_si128(__m128i __M, __m128i __V) 2710c2b4f40ac17156a424d4faa20ba5d862ecc8416Eric Christopher{ 2720c2b4f40ac17156a424d4faa20ba5d862ecc8416Eric Christopher return __builtin_ia32_ptestnzc128((__v2di)__M, (__v2di)__V); 2730c2b4f40ac17156a424d4faa20ba5d862ecc8416Eric Christopher} 2740c2b4f40ac17156a424d4faa20ba5d862ecc8416Eric Christopher 2750c2b4f40ac17156a424d4faa20ba5d862ecc8416Eric Christopher#define _mm_test_all_ones(V) _mm_testc_si128((V), _mm_cmpeq_epi32((V), (V))) 2760c2b4f40ac17156a424d4faa20ba5d862ecc8416Eric Christopher#define _mm_test_mix_ones_zeros(M, V) _mm_testnzc_si128((M), (V)) 2770d57ca1449d7a4d7afb927d4c3c1069dc6339372Bob Wilson#define _mm_test_all_zeros(M, V) _mm_testz_si128 ((M), (V)) 2780c2b4f40ac17156a424d4faa20ba5d862ecc8416Eric Christopher 2797bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher/* SSE4 64-bit Packed Integer Comparisons. */ 2801bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 2817bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher_mm_cmpeq_epi64(__m128i __V1, __m128i __V2) 2827bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher{ 2834c07c5dfebd270b2f0660e86f056eeafdb26a4fbCraig Topper return (__m128i)((__v2di)__V1 == (__v2di)__V2); 2847bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher} 2857bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher 2867bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher/* SSE4 Packed Integer Sign-Extension. */ 2871bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 2887bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher_mm_cvtepi8_epi16(__m128i __V) 2897bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher{ 2907bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher return (__m128i) __builtin_ia32_pmovsxbw128((__v16qi) __V); 2917bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher} 2927bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher 2931bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 2947bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher_mm_cvtepi8_epi32(__m128i __V) 2957bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher{ 2967bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher return (__m128i) __builtin_ia32_pmovsxbd128((__v16qi) __V); 2977bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher} 2987bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher 2991bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 3007bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher_mm_cvtepi8_epi64(__m128i __V) 3017bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher{ 3027bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher return (__m128i) __builtin_ia32_pmovsxbq128((__v16qi) __V); 3037bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher} 3047bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher 3051bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 3067bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher_mm_cvtepi16_epi32(__m128i __V) 3077bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher{ 3087bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher return (__m128i) __builtin_ia32_pmovsxwd128((__v8hi) __V); 3097bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher} 3107bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher 3111bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 3127bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher_mm_cvtepi16_epi64(__m128i __V) 3137bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher{ 3147bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher return (__m128i) __builtin_ia32_pmovsxwq128((__v8hi)__V); 3157bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher} 3167bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher 3171bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 3187bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher_mm_cvtepi32_epi64(__m128i __V) 3197bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher{ 3207bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher return (__m128i) __builtin_ia32_pmovsxdq128((__v4si)__V); 3217bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher} 3227bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher 3237bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher/* SSE4 Packed Integer Zero-Extension. */ 3241bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 3257bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher_mm_cvtepu8_epi16(__m128i __V) 3267bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher{ 3277bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher return (__m128i) __builtin_ia32_pmovzxbw128((__v16qi) __V); 3287bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher} 3297bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher 3301bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 3317bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher_mm_cvtepu8_epi32(__m128i __V) 3327bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher{ 3337bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher return (__m128i) __builtin_ia32_pmovzxbd128((__v16qi)__V); 3347bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher} 3357bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher 3361bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 3377bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher_mm_cvtepu8_epi64(__m128i __V) 3387bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher{ 3397bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher return (__m128i) __builtin_ia32_pmovzxbq128((__v16qi)__V); 3407bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher} 3417bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher 3421bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 3437bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher_mm_cvtepu16_epi32(__m128i __V) 3447bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher{ 3457bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher return (__m128i) __builtin_ia32_pmovzxwd128((__v8hi)__V); 3467bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher} 3477bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher 3481bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 3497bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher_mm_cvtepu16_epi64(__m128i __V) 3507bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher{ 3517bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher return (__m128i) __builtin_ia32_pmovzxwq128((__v8hi)__V); 3527bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher} 3537bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher 3541bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 3557bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher_mm_cvtepu32_epi64(__m128i __V) 3567bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher{ 3577bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher return (__m128i) __builtin_ia32_pmovzxdq128((__v4si)__V); 3587bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher} 3597bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher 3607bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher/* SSE4 Pack with Unsigned Saturation. */ 3611bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 3627bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher_mm_packus_epi32(__m128i __V1, __m128i __V2) 3637bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher{ 3647bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher return (__m128i) __builtin_ia32_packusdw128((__v4si)__V1, (__v4si)__V2); 3657bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher} 3667bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher 3677bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher/* SSE4 Multiple Packed Sums of Absolute Difference. */ 3685aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper#define _mm_mpsadbw_epu8(X, Y, M) __extension__ ({ \ 3695aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper __m128i __X = (X); \ 3705aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper __m128i __Y = (Y); \ 3715aeaca3fa755cddba583842e7a0c3e168bf71b4dCraig Topper (__m128i) __builtin_ia32_mpsadbw128((__v16qi)__X, (__v16qi)__Y, (M)); }) 3727bd0dfd322d7a2d6a5461dcbdb2a83fd729dd8e2Eric Christopher 3735228982f07d3420672060b2429a797b0cd3e00d5Craig Topperstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 3745228982f07d3420672060b2429a797b0cd3e00d5Craig Topper_mm_minpos_epu16(__m128i __V) 3755228982f07d3420672060b2429a797b0cd3e00d5Craig Topper{ 3765228982f07d3420672060b2429a797b0cd3e00d5Craig Topper return (__m128i) __builtin_ia32_phminposuw128((__v8hi)__V); 3775228982f07d3420672060b2429a797b0cd3e00d5Craig Topper} 3785228982f07d3420672060b2429a797b0cd3e00d5Craig Topper 37967a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher/* These definitions are normally in nmmintrin.h, but gcc puts them in here 38067a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher so we'll do the same. */ 38167a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#ifdef __SSE4_2__ 38267a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher 38367a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher/* These specify the type of data that we're comparing. */ 38467a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#define _SIDD_UBYTE_OPS 0x00 38567a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#define _SIDD_UWORD_OPS 0x01 38667a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#define _SIDD_SBYTE_OPS 0x02 38767a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#define _SIDD_SWORD_OPS 0x03 38867a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher 38967a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher/* These specify the type of comparison operation. */ 39067a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#define _SIDD_CMP_EQUAL_ANY 0x00 39167a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#define _SIDD_CMP_RANGES 0x04 39267a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#define _SIDD_CMP_EQUAL_EACH 0x08 39367a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#define _SIDD_CMP_EQUAL_ORDERED 0x0c 39467a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher 39567a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher/* These macros specify the polarity of the operation. */ 39667a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#define _SIDD_POSITIVE_POLARITY 0x00 39767a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#define _SIDD_NEGATIVE_POLARITY 0x10 39867a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#define _SIDD_MASKED_POSITIVE_POLARITY 0x20 39967a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#define _SIDD_MASKED_NEGATIVE_POLARITY 0x30 40067a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher 40167a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher/* These macros are used in _mm_cmpXstri() to specify the return. */ 40267a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#define _SIDD_LEAST_SIGNIFICANT 0x00 40367a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#define _SIDD_MOST_SIGNIFICANT 0x40 40467a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher 40567a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher/* These macros are used in _mm_cmpXstri() to specify the return. */ 40667a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#define _SIDD_BIT_MASK 0x00 40767a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#define _SIDD_UNIT_MASK 0x40 40867a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher 40967a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher/* SSE4.2 Packed Comparison Intrinsics. */ 41067a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#define _mm_cmpistrm(A, B, M) __builtin_ia32_pcmpistrm128((A), (B), (M)) 41167a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#define _mm_cmpistri(A, B, M) __builtin_ia32_pcmpistri128((A), (B), (M)) 41267a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher 41367a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#define _mm_cmpestrm(A, LA, B, LB, M) \ 41467a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher __builtin_ia32_pcmpestrm128((A), (LA), (B), (LB), (M)) 4154b45c111f0ac43441c475aca4ecc1d62499e0cb8Chandler Carruth#define _mm_cmpestri(A, LA, B, LB, M) \ 41667a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher __builtin_ia32_pcmpestri128((A), (LA), (B), (LB), (M)) 41767a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher 41867a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher/* SSE4.2 Packed Comparison Intrinsics and EFlag Reading. */ 419355130fde923c7284d32cc61fc3a34673bf14852Eli Friedman#define _mm_cmpistra(A, B, M) \ 420355130fde923c7284d32cc61fc3a34673bf14852Eli Friedman __builtin_ia32_pcmpistria128((A), (B), (M)) 421355130fde923c7284d32cc61fc3a34673bf14852Eli Friedman#define _mm_cmpistrc(A, B, M) \ 422355130fde923c7284d32cc61fc3a34673bf14852Eli Friedman __builtin_ia32_pcmpistric128((A), (B), (M)) 423355130fde923c7284d32cc61fc3a34673bf14852Eli Friedman#define _mm_cmpistro(A, B, M) \ 424355130fde923c7284d32cc61fc3a34673bf14852Eli Friedman __builtin_ia32_pcmpistrio128((A), (B), (M)) 425355130fde923c7284d32cc61fc3a34673bf14852Eli Friedman#define _mm_cmpistrs(A, B, M) \ 426355130fde923c7284d32cc61fc3a34673bf14852Eli Friedman __builtin_ia32_pcmpistris128((A), (B), (M)) 427355130fde923c7284d32cc61fc3a34673bf14852Eli Friedman#define _mm_cmpistrz(A, B, M) \ 428355130fde923c7284d32cc61fc3a34673bf14852Eli Friedman __builtin_ia32_pcmpistriz128((A), (B), (M)) 42967a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher 43067a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#define _mm_cmpestra(A, LA, B, LB, M) \ 43167a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher __builtin_ia32_pcmpestria128((A), (LA), (B), (LB), (M)) 43267a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#define _mm_cmpestrc(A, LA, B, LB, M) \ 43367a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher __builtin_ia32_pcmpestric128((A), (LA), (B), (LB), (M)) 43467a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#define _mm_cmpestro(A, LA, B, LB, M) \ 43567a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher __builtin_ia32_pcmpestrio128((A), (LA), (B), (LB), (M)) 43667a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#define _mm_cmpestrs(A, LA, B, LB, M) \ 43767a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher __builtin_ia32_pcmpestris128((A), (LA), (B), (LB), (M)) 43867a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#define _mm_cmpestrz(A, LA, B, LB, M) \ 43967a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher __builtin_ia32_pcmpestriz128((A), (LA), (B), (LB), (M)) 44067a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher 44167a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher/* SSE4.2 Compare Packed Data -- Greater Than. */ 4421bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) 44367a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher_mm_cmpgt_epi64(__m128i __V1, __m128i __V2) 44467a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher{ 4454c07c5dfebd270b2f0660e86f056eeafdb26a4fbCraig Topper return (__m128i)((__v2di)__V1 > (__v2di)__V2); 44667a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher} 44767a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher 44867a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher/* SSE4.2 Accumulate CRC32. */ 4491bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ unsigned int __attribute__((__always_inline__, __nodebug__)) 45067a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher_mm_crc32_u8(unsigned int __C, unsigned char __D) 45167a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher{ 45267a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher return __builtin_ia32_crc32qi(__C, __D); 45367a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher} 45467a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher 4551bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ unsigned int __attribute__((__always_inline__, __nodebug__)) 45667a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher_mm_crc32_u16(unsigned int __C, unsigned short __D) 45767a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher{ 45867a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher return __builtin_ia32_crc32hi(__C, __D); 45967a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher} 46067a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher 4611bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ unsigned int __attribute__((__always_inline__, __nodebug__)) 46267a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher_mm_crc32_u32(unsigned int __C, unsigned int __D) 46367a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher{ 46467a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher return __builtin_ia32_crc32si(__C, __D); 46567a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher} 46667a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher 46767a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#ifdef __x86_64__ 4681bddbcbd11de09c7bcb48c3d661c447967db73a7Chris Lattnerstatic __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__)) 46967a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher_mm_crc32_u64(unsigned long long __C, unsigned long long __D) 47067a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher{ 47167a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher return __builtin_ia32_crc32di(__C, __D); 47267a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher} 47367a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#endif /* __x86_64__ */ 47467a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher 475e14e08b2552b43d81a295401ddf9b8da2c499233Craig Topper#ifdef __POPCNT__ 476e14e08b2552b43d81a295401ddf9b8da2c499233Craig Topper#include <popcntintrin.h> 477e14e08b2552b43d81a295401ddf9b8da2c499233Craig Topper#endif 47867a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher 47967a5936a1d60fff1ec12441de63fd27ea50b0130Eric Christopher#endif /* __SSE4_2__ */ 480b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#endif /* __SSE4_1__ */ 481b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher 482b71f95647189f3b0f7f5f5a39c8c3ab8130dcf1fEric Christopher#endif /* _SMMINTRIN_H */ 483