1c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// This file is part of Eigen, a lightweight C++ template library 2c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// for linear algebra. 3c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// 4c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// Copyright (C) 2008-2009 Gael Guennebaud <gael.guennebaud@inria.fr> 5c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// 6c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// This Source Code Form is subject to the terms of the Mozilla 7c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// Public License v. 2.0. If a copy of the MPL was not distributed 8c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 10c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#ifndef EIGEN_PACKET_MATH_SSE_H 11c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#define EIGEN_PACKET_MATH_SSE_H 12c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 13c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathnamespace Eigen { 14c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 15c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathnamespace internal { 16c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 17c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 18c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8 19c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#endif 20c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 21c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 22c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS (2*sizeof(void*)) 23c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#endif 24c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 252b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifdef __FMA__ 262b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD 272b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD 1 282b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 292b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 302b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 312b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#if (defined EIGEN_VECTORIZE_AVX) && (EIGEN_COMP_GNUC_STRICT || EIGEN_COMP_MINGW) && (__GXX_ABI_VERSION < 1004) 322b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// With GCC's default ABI version, a __m128 or __m256 are the same types and therefore we cannot 332b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// have overloads for both types without linking error. 342b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// One solution is to increase ABI version using -fabi-version=4 (or greater). 352b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// Otherwise, we workaround this inconvenience by wrapping 128bit types into the following helper 362b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// structure: 372b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<typename T> 382b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangstruct eigen_packet_wrapper 392b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 402b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_ALWAYS_INLINE operator T&() { return m_val; } 412b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_ALWAYS_INLINE operator const T&() const { return m_val; } 422b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_ALWAYS_INLINE eigen_packet_wrapper() {} 432b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_ALWAYS_INLINE eigen_packet_wrapper(const T &v) : m_val(v) {} 442b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_ALWAYS_INLINE eigen_packet_wrapper& operator=(const T &v) { 452b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang m_val = v; 462b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return *this; 472b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang } 482b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 492b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang T m_val; 502b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang}; 512b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtypedef eigen_packet_wrapper<__m128> Packet4f; 522b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtypedef eigen_packet_wrapper<__m128i> Packet4i; 532b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtypedef eigen_packet_wrapper<__m128d> Packet2d; 542b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#else 55c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtypedef __m128 Packet4f; 56c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtypedef __m128i Packet4i; 57c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtypedef __m128d Packet2d; 582b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 59c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 60c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> struct is_arithmetic<__m128> { enum { value = true }; }; 61c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> struct is_arithmetic<__m128i> { enum { value = true }; }; 62c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> struct is_arithmetic<__m128d> { enum { value = true }; }; 63c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 64c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#define vec4f_swizzle1(v,p,q,r,s) \ 65c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath (_mm_castsi128_ps(_mm_shuffle_epi32( _mm_castps_si128(v), ((s)<<6|(r)<<4|(q)<<2|(p))))) 66c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 67c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#define vec4i_swizzle1(v,p,q,r,s) \ 68c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath (_mm_shuffle_epi32( v, ((s)<<6|(r)<<4|(q)<<2|(p)))) 69c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 70c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#define vec2d_swizzle1(v,p,q) \ 71c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath (_mm_castsi128_pd(_mm_shuffle_epi32( _mm_castpd_si128(v), ((q*2+1)<<6|(q*2)<<4|(p*2+1)<<2|(p*2))))) 72c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 73c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#define vec4f_swizzle2(a,b,p,q,r,s) \ 74c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath (_mm_shuffle_ps( (a), (b), ((s)<<6|(r)<<4|(q)<<2|(p)))) 75c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 76c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#define vec4i_swizzle2(a,b,p,q,r,s) \ 77c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath (_mm_castps_si128( (_mm_shuffle_ps( _mm_castsi128_ps(a), _mm_castsi128_ps(b), ((s)<<6|(r)<<4|(q)<<2|(p)))))) 78c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 79c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \ 80c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath const Packet4f p4f_##NAME = pset1<Packet4f>(X) 81c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 827faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez#define _EIGEN_DECLARE_CONST_Packet2d(NAME,X) \ 837faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez const Packet2d p2d_##NAME = pset1<Packet2d>(X) 847faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez 85c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \ 86c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath const Packet4f p4f_##NAME = _mm_castsi128_ps(pset1<Packet4i>(X)) 87c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 88c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \ 89c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath const Packet4i p4i_##NAME = pset1<Packet4i>(X) 90c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 91c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 922b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// Use the packet_traits defined in AVX/PacketMath.h instead if we're going 932b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// to leverage AVX instructions. 942b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifndef EIGEN_VECTORIZE_AVX 95c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> struct packet_traits<float> : default_packet_traits 96c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 97c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath typedef Packet4f type; 982b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang typedef Packet4f half; 99c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath enum { 100c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath Vectorizable = 1, 101c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath AlignedOnScalar = 1, 102c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath size=4, 1032b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasHalfPacket = 0, 104c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 1057faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez HasDiv = 1, 106c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath HasSin = EIGEN_FAST_MATH, 107c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath HasCos = EIGEN_FAST_MATH, 108c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath HasLog = 1, 109c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath HasExp = 1, 1102b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasSqrt = 1, 1112b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasRsqrt = 1, 1122b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasTanh = EIGEN_FAST_MATH, 1132b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasBlend = 1 1142b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 1152b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifdef EIGEN_VECTORIZE_SSE4_1 1162b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang , 1172b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasRound = 1, 1182b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasFloor = 1, 1192b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasCeil = 1 1202b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 121c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath }; 122c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath}; 123c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> struct packet_traits<double> : default_packet_traits 124c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 125c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath typedef Packet2d type; 1262b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang typedef Packet2d half; 127c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath enum { 128c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath Vectorizable = 1, 129c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath AlignedOnScalar = 1, 130c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath size=2, 1312b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasHalfPacket = 0, 132c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 1337faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez HasDiv = 1, 1347faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez HasExp = 1, 1352b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasSqrt = 1, 1362b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasRsqrt = 1, 1372b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasBlend = 1 1382b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 1392b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifdef EIGEN_VECTORIZE_SSE4_1 1402b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang , 1412b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasRound = 1, 1422b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasFloor = 1, 1432b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasCeil = 1 1442b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 145c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath }; 146c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath}; 1472b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 148c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> struct packet_traits<int> : default_packet_traits 149c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 150c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath typedef Packet4i type; 1512b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang typedef Packet4i half; 152c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath enum { 153c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath Vectorizable = 1, 154c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath AlignedOnScalar = 1, 1552b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang size=4, 1562b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 1572b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasBlend = 1 158c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath }; 159c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath}; 160c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 1612b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> struct unpacket_traits<Packet4f> { typedef float type; enum {size=4, alignment=Aligned16}; typedef Packet4f half; }; 1622b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> struct unpacket_traits<Packet2d> { typedef double type; enum {size=2, alignment=Aligned16}; typedef Packet2d half; }; 1632b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> struct unpacket_traits<Packet4i> { typedef int type; enum {size=4, alignment=Aligned16}; typedef Packet4i half; }; 164c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 1652b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifndef EIGEN_VECTORIZE_AVX 1662b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> struct scalar_div_cost<float,true> { enum { value = 7 }; }; 1672b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> struct scalar_div_cost<double,true> { enum { value = 8 }; }; 1682b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 1692b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 1702b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#if EIGEN_COMP_MSVC==1500 171c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// Workaround MSVC 9 internal compiler error. 172c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// TODO: It has been detected with win64 builds (amd64), so let's check whether it also happens in 32bits+SSE mode 173c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// TODO: let's check whether there does not exist a better fix, like adding a pset0() function. (it crashed on pset1(0)). 174c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) { return _mm_set_ps(from,from,from,from); } 175c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) { return _mm_set_pd(from,from); } 176c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int& from) { return _mm_set_epi32(from,from,from,from); } 177c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#else 1782b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) { return _mm_set_ps1(from); } 179c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) { return _mm_set1_pd(from); } 180c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int& from) { return _mm_set1_epi32(from); } 181c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#endif 182c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 1832b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// GCC generates a shufps instruction for _mm_set1_ps/_mm_load1_ps instead of the more efficient pshufd instruction. 1842b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// However, using inrinsics for pset1 makes gcc to generate crappy code in some cases (see bug 203) 1852b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// Using inline assembly is also not an option because then gcc fails to reorder properly the instructions. 1862b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// Therefore, we introduced the pload1 functions to be used in product kernels for which bug 203 does not apply. 1872b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// Also note that with AVX, we want it to generate a vbroadcastss. 1882b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#if EIGEN_COMP_GNUC_STRICT && (!defined __AVX__) 1892b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4f pload1<Packet4f>(const float *from) { 1902b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return vec4f_swizzle1(_mm_load_ss(from),0,0,0,0); 1912b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 1922b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 1932b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 1942b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(const float& a) { return _mm_add_ps(pset1<Packet4f>(a), _mm_set_ps(3,2,1,0)); } 1952b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(const double& a) { return _mm_add_pd(pset1<Packet2d>(a),_mm_set_pd(1,0)); } 1962b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(const int& a) { return _mm_add_epi32(pset1<Packet4i>(a),_mm_set_epi32(3,2,1,0)); } 197c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 198c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_add_ps(a,b); } 199c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_add_pd(a,b); } 200c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_add_epi32(a,b); } 201c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 202c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_sub_ps(a,b); } 203c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_sub_pd(a,b); } 204c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_sub_epi32(a,b); } 205c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 206c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) 207c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 208c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath const Packet4f mask = _mm_castsi128_ps(_mm_setr_epi32(0x80000000,0x80000000,0x80000000,0x80000000)); 209c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return _mm_xor_ps(a,mask); 210c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 211c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a) 212c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 213c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath const Packet2d mask = _mm_castsi128_pd(_mm_setr_epi32(0x0,0x80000000,0x0,0x80000000)); 214c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return _mm_xor_pd(a,mask); 215c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 216c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) 217c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 2182b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return psub(Packet4i(_mm_setr_epi32(0,0,0,0)), a); 219c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 220c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 2217faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandeztemplate<> EIGEN_STRONG_INLINE Packet4f pconj(const Packet4f& a) { return a; } 2227faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandeztemplate<> EIGEN_STRONG_INLINE Packet2d pconj(const Packet2d& a) { return a; } 2237faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandeztemplate<> EIGEN_STRONG_INLINE Packet4i pconj(const Packet4i& a) { return a; } 2247faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez 225c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_mul_ps(a,b); } 226c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_mul_pd(a,b); } 227c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b) 228c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 229c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#ifdef EIGEN_VECTORIZE_SSE4_1 230c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return _mm_mullo_epi32(a,b); 231c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#else 232c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // this version is slightly faster than 4 scalar products 233c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return vec4i_swizzle1( 234c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath vec4i_swizzle2( 235c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath _mm_mul_epu32(a,b), 236c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath _mm_mul_epu32(vec4i_swizzle1(a,1,0,3,2), 237c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath vec4i_swizzle1(b,1,0,3,2)), 238c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 0,2,0,2), 239c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 0,2,1,3); 240c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#endif 241c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 242c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 243c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_div_ps(a,b); } 244c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_div_pd(a,b); } 245c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 246c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// for some weird raisons, it has to be overloaded for packet of integers 247c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) { return padd(pmul(a,b), c); } 2482b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifdef __FMA__ 2492b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) { return _mm_fmadd_ps(a,b,c); } 2502b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) { return _mm_fmadd_pd(a,b,c); } 2512b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 252c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 253c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_min_ps(a,b); } 254c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_min_pd(a,b); } 255c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b) 256c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 2577faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez#ifdef EIGEN_VECTORIZE_SSE4_1 2587faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez return _mm_min_epi32(a,b); 2597faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez#else 260c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // after some bench, this version *is* faster than a scalar implementation 261c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath Packet4i mask = _mm_cmplt_epi32(a,b); 262c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return _mm_or_si128(_mm_and_si128(mask,a),_mm_andnot_si128(mask,b)); 2637faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez#endif 264c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 265c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 266c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_max_ps(a,b); } 267c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_max_pd(a,b); } 268c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b) 269c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 2707faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez#ifdef EIGEN_VECTORIZE_SSE4_1 2717faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez return _mm_max_epi32(a,b); 2727faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez#else 273c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // after some bench, this version *is* faster than a scalar implementation 274c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath Packet4i mask = _mm_cmpgt_epi32(a,b); 275c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return _mm_or_si128(_mm_and_si128(mask,a),_mm_andnot_si128(mask,b)); 2767faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez#endif 277c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 278c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 2792b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifdef EIGEN_VECTORIZE_SSE4_1 2802b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4f pround<Packet4f>(const Packet4f& a) { return _mm_round_ps(a, 0); } 2812b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d pround<Packet2d>(const Packet2d& a) { return _mm_round_pd(a, 0); } 2822b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 2832b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(const Packet4f& a) { return _mm_ceil_ps(a); } 2842b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d pceil<Packet2d>(const Packet2d& a) { return _mm_ceil_pd(a); } 2852b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 2862b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a) { return _mm_floor_ps(a); } 2872b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(const Packet2d& a) { return _mm_floor_pd(a); } 2882b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 2892b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 290c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_and_ps(a,b); } 291c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_and_pd(a,b); } 292c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_and_si128(a,b); } 293c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 294c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f por<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_or_ps(a,b); } 295c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet2d por<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_or_pd(a,b); } 296c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i por<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_or_si128(a,b); } 297c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 298c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_xor_ps(a,b); } 299c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_xor_pd(a,b); } 300c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_xor_si128(a,b); } 301c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 302c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_andnot_ps(a,b); } 303c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_andnot_pd(a,b); } 304c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_andnot_si128(a,b); } 305c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 3062b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_ps(from); } 3072b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_pd(from); } 3082b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_si128(reinterpret_cast<const __m128i*>(from)); } 309c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 3102b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#if EIGEN_COMP_MSVC 311c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from) { 312c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath EIGEN_DEBUG_UNALIGNED_LOAD 3132b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang #if (EIGEN_COMP_MSVC==1600) 314c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // NOTE Some version of MSVC10 generates bad code when using _mm_loadu_ps 315c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // (i.e., it does not generate an unaligned load!! 316c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath __m128 res = _mm_loadl_pi(_mm_set1_ps(0.0f), (const __m64*)(from)); 317c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath res = _mm_loadh_pi(res, (const __m64*)(from+2)); 318c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return res; 319c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath #else 320c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return _mm_loadu_ps(from); 321c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath #endif 322c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath } 323c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#else 324c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// NOTE: with the code below, MSVC's compiler crashes! 325c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 326c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from) 327c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 328c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath EIGEN_DEBUG_UNALIGNED_LOAD 329c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return _mm_loadu_ps(from); 330c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 3312b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 3322b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 333c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from) 334c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 335c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath EIGEN_DEBUG_UNALIGNED_LOAD 336c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return _mm_loadu_pd(from); 337c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 338c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int* from) 339c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 340c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath EIGEN_DEBUG_UNALIGNED_LOAD 3412b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return _mm_loadu_si128(reinterpret_cast<const __m128i*>(from)); 342c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 3432b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 344c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 345c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from) 346c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 347c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return vec4f_swizzle1(_mm_castpd_ps(_mm_load_sd(reinterpret_cast<const double*>(from))), 0, 0, 1, 1); 348c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 349c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(const double* from) 350c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ return pset1<Packet2d>(from[0]); } 351c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int* from) 352c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 353c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath Packet4i tmp; 3542b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang tmp = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(from)); 355c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return vec4i_swizzle1(tmp, 0, 0, 1, 1); 356c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 357c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 3582b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet4f& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_ps(to, from); } 3592b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet2d& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_pd(to, from); } 3602b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE void pstore<int>(int* to, const Packet4i& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_si128(reinterpret_cast<__m128i*>(to), from); } 3612b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 3622b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet2d& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm_storeu_pd(to, from); } 3632b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet4f& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm_storeu_ps(to, from); } 3642b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet4i& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm_storeu_si128(reinterpret_cast<__m128i*>(to), from); } 365c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 3662b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_DEVICE_FUNC inline Packet4f pgather<float, Packet4f>(const float* from, Index stride) 3672b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 3682b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return _mm_set_ps(from[3*stride], from[2*stride], from[1*stride], from[0*stride]); 3692b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 3702b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_DEVICE_FUNC inline Packet2d pgather<double, Packet2d>(const double* from, Index stride) 3712b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 3722b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return _mm_set_pd(from[1*stride], from[0*stride]); 3732b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 3742b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_DEVICE_FUNC inline Packet4i pgather<int, Packet4i>(const int* from, Index stride) 3752b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 3762b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return _mm_set_epi32(from[3*stride], from[2*stride], from[1*stride], from[0*stride]); 3772b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang } 3782b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 3792b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_DEVICE_FUNC inline void pscatter<float, Packet4f>(float* to, const Packet4f& from, Index stride) 3802b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 3812b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang to[stride*0] = _mm_cvtss_f32(from); 3822b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang to[stride*1] = _mm_cvtss_f32(_mm_shuffle_ps(from, from, 1)); 3832b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang to[stride*2] = _mm_cvtss_f32(_mm_shuffle_ps(from, from, 2)); 3842b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang to[stride*3] = _mm_cvtss_f32(_mm_shuffle_ps(from, from, 3)); 3852b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 3862b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_DEVICE_FUNC inline void pscatter<double, Packet2d>(double* to, const Packet2d& from, Index stride) 3872b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 3882b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang to[stride*0] = _mm_cvtsd_f64(from); 3892b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang to[stride*1] = _mm_cvtsd_f64(_mm_shuffle_pd(from, from, 1)); 3902b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 3912b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_DEVICE_FUNC inline void pscatter<int, Packet4i>(int* to, const Packet4i& from, Index stride) 3922b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 3932b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang to[stride*0] = _mm_cvtsi128_si32(from); 3942b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang to[stride*1] = _mm_cvtsi128_si32(_mm_shuffle_epi32(from, 1)); 3952b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang to[stride*2] = _mm_cvtsi128_si32(_mm_shuffle_epi32(from, 2)); 3962b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang to[stride*3] = _mm_cvtsi128_si32(_mm_shuffle_epi32(from, 3)); 397c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 398c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 399c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// some compilers might be tempted to perform multiple moves instead of using a vector path. 400c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE void pstore1<Packet4f>(float* to, const float& a) 401c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 402c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath Packet4f pa = _mm_set_ss(a); 4032b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang pstore(to, Packet4f(vec4f_swizzle1(pa,0,0,0,0))); 404c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 405c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// some compilers might be tempted to perform multiple moves instead of using a vector path. 406c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE void pstore1<Packet2d>(double* to, const double& a) 407c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 408c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath Packet2d pa = _mm_set_sd(a); 4092b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang pstore(to, Packet2d(vec2d_swizzle1(pa,0,0))); 410c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 411c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 4122b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifndef EIGEN_VECTORIZE_AVX 413c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); } 414c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); } 415c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); } 4162b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 417c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 4182b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#if EIGEN_COMP_MSVC_STRICT && EIGEN_OS_WIN64 419c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// The temporary variable fixes an internal compilation error in vs <= 2008 and a wrong-result bug in vs 2010 420c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// Direct of the struct members fixed bug #62. 421c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { return a.m128_f32[0]; } 422c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { return a.m128d_f64[0]; } 423c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) { int x = _mm_cvtsi128_si32(a); return x; } 4242b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#elif EIGEN_COMP_MSVC_STRICT 425c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// The temporary variable fixes an internal compilation error in vs <= 2008 and a wrong-result bug in vs 2010 426c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { float x = _mm_cvtss_f32(a); return x; } 427c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { double x = _mm_cvtsd_f64(a); return x; } 428c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) { int x = _mm_cvtsi128_si32(a); return x; } 429c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#else 430c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { return _mm_cvtss_f32(a); } 431c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { return _mm_cvtsd_f64(a); } 432c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) { return _mm_cvtsi128_si32(a); } 433c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#endif 434c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 435c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) 436c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ return _mm_shuffle_ps(a,a,0x1B); } 437c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a) 438c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ return _mm_shuffle_pd(a,a,0x1); } 439c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a) 440c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ return _mm_shuffle_epi32(a,0x1B); } 441c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 442c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a) 443c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 444c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath const Packet4f mask = _mm_castsi128_ps(_mm_setr_epi32(0x7FFFFFFF,0x7FFFFFFF,0x7FFFFFFF,0x7FFFFFFF)); 445c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return _mm_and_ps(a,mask); 446c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 447c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) 448c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 449c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath const Packet2d mask = _mm_castsi128_pd(_mm_setr_epi32(0xFFFFFFFF,0x7FFFFFFF,0xFFFFFFFF,0x7FFFFFFF)); 450c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return _mm_and_pd(a,mask); 451c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 452c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) 453c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 454c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath #ifdef EIGEN_VECTORIZE_SSSE3 455c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return _mm_abs_epi32(a); 456c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath #else 457c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath Packet4i aux = _mm_srai_epi32(a,31); 458c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return _mm_sub_epi32(_mm_xor_si128(a,aux),aux); 459c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath #endif 460c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 461c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 4622b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// with AVX, the default implementations based on pload1 are faster 4632b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifndef __AVX__ 4642b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE void 4652b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangpbroadcast4<Packet4f>(const float *a, 4662b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3) 4672b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 4682b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang a3 = pload<Packet4f>(a); 4692b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang a0 = vec4f_swizzle1(a3, 0,0,0,0); 4702b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang a1 = vec4f_swizzle1(a3, 1,1,1,1); 4712b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang a2 = vec4f_swizzle1(a3, 2,2,2,2); 4722b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang a3 = vec4f_swizzle1(a3, 3,3,3,3); 4732b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 4742b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE void 4752b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangpbroadcast4<Packet2d>(const double *a, 4762b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Packet2d& a0, Packet2d& a1, Packet2d& a2, Packet2d& a3) 4772b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 4782b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifdef EIGEN_VECTORIZE_SSE3 4792b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang a0 = _mm_loaddup_pd(a+0); 4802b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang a1 = _mm_loaddup_pd(a+1); 4812b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang a2 = _mm_loaddup_pd(a+2); 4822b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang a3 = _mm_loaddup_pd(a+3); 4832b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#else 4842b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang a1 = pload<Packet2d>(a); 4852b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang a0 = vec2d_swizzle1(a1, 0,0); 4862b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang a1 = vec2d_swizzle1(a1, 1,1); 4872b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang a3 = pload<Packet2d>(a+2); 4882b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang a2 = vec2d_swizzle1(a3, 0,0); 4892b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang a3 = vec2d_swizzle1(a3, 1,1); 4902b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 4912b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 4922b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 4932b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 494c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan KamathEIGEN_STRONG_INLINE void punpackp(Packet4f* vecs) 495c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 496c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath vecs[1] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0x55)); 497c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath vecs[2] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0xAA)); 498c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath vecs[3] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0xFF)); 499c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath vecs[0] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0x00)); 500c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 501c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 502c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#ifdef EIGEN_VECTORIZE_SSE3 503c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(const Packet4f* vecs) 504c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 505c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return _mm_hadd_ps(_mm_hadd_ps(vecs[0], vecs[1]),_mm_hadd_ps(vecs[2], vecs[3])); 506c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 5072b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 508c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet2d preduxp<Packet2d>(const Packet2d* vecs) 509c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 510c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return _mm_hadd_pd(vecs[0], vecs[1]); 511c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 512c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 513c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#else 514c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(const Packet4f* vecs) 515c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 516c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath Packet4f tmp0, tmp1, tmp2; 517c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath tmp0 = _mm_unpacklo_ps(vecs[0], vecs[1]); 518c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath tmp1 = _mm_unpackhi_ps(vecs[0], vecs[1]); 519c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath tmp2 = _mm_unpackhi_ps(vecs[2], vecs[3]); 520c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath tmp0 = _mm_add_ps(tmp0, tmp1); 521c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath tmp1 = _mm_unpacklo_ps(vecs[2], vecs[3]); 522c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath tmp1 = _mm_add_ps(tmp1, tmp2); 523c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath tmp2 = _mm_movehl_ps(tmp1, tmp0); 524c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath tmp0 = _mm_movelh_ps(tmp0, tmp1); 525c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return _mm_add_ps(tmp0, tmp2); 526c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 527c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 528c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet2d preduxp<Packet2d>(const Packet2d* vecs) 529c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 530c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return _mm_add_pd(_mm_unpacklo_pd(vecs[0], vecs[1]), _mm_unpackhi_pd(vecs[0], vecs[1])); 531c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 532c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#endif // SSE3 533c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 5342b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a) 5352b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 5362b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang // Disable SSE3 _mm_hadd_pd that is extremely slow on all existing Intel's architectures 5372b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang // (from Nehalem to Haswell) 5382b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// #ifdef EIGEN_VECTORIZE_SSE3 5392b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// Packet4f tmp = _mm_add_ps(a, vec4f_swizzle1(a,2,3,2,3)); 5402b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// return pfirst<Packet4f>(_mm_hadd_ps(tmp, tmp)); 5412b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// #else 5422b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Packet4f tmp = _mm_add_ps(a, _mm_movehl_ps(a,a)); 5432b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return pfirst<Packet4f>(_mm_add_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1))); 5442b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// #endif 5452b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 5462b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 5472b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a) 5482b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 5492b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang // Disable SSE3 _mm_hadd_pd that is extremely slow on all existing Intel's architectures 5502b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang // (from Nehalem to Haswell) 5512b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// #ifdef EIGEN_VECTORIZE_SSE3 5522b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// return pfirst<Packet2d>(_mm_hadd_pd(a, a)); 5532b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// #else 5542b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return pfirst<Packet2d>(_mm_add_sd(a, _mm_unpackhi_pd(a,a))); 5552b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// #endif 5562b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 5572b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 5582b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifdef EIGEN_VECTORIZE_SSSE3 5592b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4i preduxp<Packet4i>(const Packet4i* vecs) 5602b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 5612b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return _mm_hadd_epi32(_mm_hadd_epi32(vecs[0], vecs[1]),_mm_hadd_epi32(vecs[2], vecs[3])); 5622b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 5632b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE int predux<Packet4i>(const Packet4i& a) 5642b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 5652b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Packet4i tmp0 = _mm_hadd_epi32(a,a); 5662b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return pfirst<Packet4i>(_mm_hadd_epi32(tmp0,tmp0)); 5672b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 5682b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#else 569c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE int predux<Packet4i>(const Packet4i& a) 570c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 571c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath Packet4i tmp = _mm_add_epi32(a, _mm_unpackhi_epi64(a,a)); 5722b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return pfirst(tmp) + pfirst<Packet4i>(_mm_shuffle_epi32(tmp, 1)); 573c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 574c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 575c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i preduxp<Packet4i>(const Packet4i* vecs) 576c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 577c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath Packet4i tmp0, tmp1, tmp2; 578c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath tmp0 = _mm_unpacklo_epi32(vecs[0], vecs[1]); 579c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath tmp1 = _mm_unpackhi_epi32(vecs[0], vecs[1]); 580c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath tmp2 = _mm_unpackhi_epi32(vecs[2], vecs[3]); 581c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath tmp0 = _mm_add_epi32(tmp0, tmp1); 582c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath tmp1 = _mm_unpacklo_epi32(vecs[2], vecs[3]); 583c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath tmp1 = _mm_add_epi32(tmp1, tmp2); 584c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath tmp2 = _mm_unpacklo_epi64(tmp0, tmp1); 585c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath tmp0 = _mm_unpackhi_epi64(tmp0, tmp1); 586c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return _mm_add_epi32(tmp0, tmp2); 587c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 5882b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 589c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// Other reduction functions: 590c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 591c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// mul 592c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a) 593c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 594c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath Packet4f tmp = _mm_mul_ps(a, _mm_movehl_ps(a,a)); 5952b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return pfirst<Packet4f>(_mm_mul_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1))); 596c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 597c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a) 598c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 5992b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return pfirst<Packet2d>(_mm_mul_sd(a, _mm_unpackhi_pd(a,a))); 600c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 601c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE int predux_mul<Packet4i>(const Packet4i& a) 602c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 603c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // after some experiments, it is seems this is the fastest way to implement it 604c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // for GCC (eg., reusing pmul is very slow !) 605c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // TODO try to call _mm_mul_epu32 directly 606c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath EIGEN_ALIGN16 int aux[4]; 607c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath pstore(aux, a); 608c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return (aux[0] * aux[1]) * (aux[2] * aux[3]);; 609c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 610c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 611c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// min 612c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a) 613c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 614c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath Packet4f tmp = _mm_min_ps(a, _mm_movehl_ps(a,a)); 6152b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return pfirst<Packet4f>(_mm_min_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1))); 616c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 617c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE double predux_min<Packet2d>(const Packet2d& a) 618c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 6192b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return pfirst<Packet2d>(_mm_min_sd(a, _mm_unpackhi_pd(a,a))); 620c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 621c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE int predux_min<Packet4i>(const Packet4i& a) 622c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 6232b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifdef EIGEN_VECTORIZE_SSE4_1 6242b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Packet4i tmp = _mm_min_epi32(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(0,0,3,2))); 6252b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return pfirst<Packet4i>(_mm_min_epi32(tmp,_mm_shuffle_epi32(tmp, 1))); 6262b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#else 627c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // after some experiments, it is seems this is the fastest way to implement it 628c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // for GCC (eg., it does not like using std::min after the pstore !!) 629c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath EIGEN_ALIGN16 int aux[4]; 630c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath pstore(aux, a); 6317faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez int aux0 = aux[0]<aux[1] ? aux[0] : aux[1]; 6327faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez int aux2 = aux[2]<aux[3] ? aux[2] : aux[3]; 633c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return aux0<aux2 ? aux0 : aux2; 6342b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif // EIGEN_VECTORIZE_SSE4_1 635c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 636c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 637c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// max 638c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a) 639c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 640c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath Packet4f tmp = _mm_max_ps(a, _mm_movehl_ps(a,a)); 6412b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return pfirst<Packet4f>(_mm_max_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1))); 642c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 643c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE double predux_max<Packet2d>(const Packet2d& a) 644c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 6452b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return pfirst<Packet2d>(_mm_max_sd(a, _mm_unpackhi_pd(a,a))); 646c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 647c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE int predux_max<Packet4i>(const Packet4i& a) 648c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 6492b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifdef EIGEN_VECTORIZE_SSE4_1 6502b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Packet4i tmp = _mm_max_epi32(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(0,0,3,2))); 6512b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return pfirst<Packet4i>(_mm_max_epi32(tmp,_mm_shuffle_epi32(tmp, 1))); 6522b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#else 653c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // after some experiments, it is seems this is the fastest way to implement it 654c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // for GCC (eg., it does not like using std::min after the pstore !!) 655c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath EIGEN_ALIGN16 int aux[4]; 656c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath pstore(aux, a); 6577faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez int aux0 = aux[0]>aux[1] ? aux[0] : aux[1]; 6587faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez int aux2 = aux[2]>aux[3] ? aux[2] : aux[3]; 659c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return aux0>aux2 ? aux0 : aux2; 6602b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif // EIGEN_VECTORIZE_SSE4_1 661c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 662c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 6632b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#if EIGEN_COMP_GNUC 664c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// template <> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) 665c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// { 666c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// Packet4f res = b; 667c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// asm("mulps %[a], %[b] \n\taddps %[c], %[b]" : [b] "+x" (res) : [a] "x" (a), [c] "x" (c)); 668c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// return res; 669c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// } 670c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// EIGEN_STRONG_INLINE Packet4i _mm_alignr_epi8(const Packet4i& a, const Packet4i& b, const int i) 671c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// { 672c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// Packet4i res = a; 673c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// asm("palignr %[i], %[a], %[b] " : [b] "+x" (res) : [a] "x" (a), [i] "i" (i)); 674c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// return res; 675c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// } 676c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#endif 677c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 678c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#ifdef EIGEN_VECTORIZE_SSSE3 679c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// SSSE3 versions 680c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<int Offset> 681c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathstruct palign_impl<Offset,Packet4f> 682c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 683c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath static EIGEN_STRONG_INLINE void run(Packet4f& first, const Packet4f& second) 684c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath { 685c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath if (Offset!=0) 686c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath first = _mm_castsi128_ps(_mm_alignr_epi8(_mm_castps_si128(second), _mm_castps_si128(first), Offset*4)); 687c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath } 688c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath}; 689c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 690c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<int Offset> 691c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathstruct palign_impl<Offset,Packet4i> 692c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 693c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath static EIGEN_STRONG_INLINE void run(Packet4i& first, const Packet4i& second) 694c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath { 695c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath if (Offset!=0) 696c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath first = _mm_alignr_epi8(second,first, Offset*4); 697c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath } 698c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath}; 699c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 700c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<int Offset> 701c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathstruct palign_impl<Offset,Packet2d> 702c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 703c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath static EIGEN_STRONG_INLINE void run(Packet2d& first, const Packet2d& second) 704c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath { 705c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath if (Offset==1) 706c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath first = _mm_castsi128_pd(_mm_alignr_epi8(_mm_castpd_si128(second), _mm_castpd_si128(first), 8)); 707c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath } 708c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath}; 709c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#else 710c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// SSE2 versions 711c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<int Offset> 712c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathstruct palign_impl<Offset,Packet4f> 713c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 714c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath static EIGEN_STRONG_INLINE void run(Packet4f& first, const Packet4f& second) 715c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath { 716c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath if (Offset==1) 717c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath { 718c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath first = _mm_move_ss(first,second); 719c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath first = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(first),0x39)); 720c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath } 721c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath else if (Offset==2) 722c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath { 723c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath first = _mm_movehl_ps(first,first); 724c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath first = _mm_movelh_ps(first,second); 725c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath } 726c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath else if (Offset==3) 727c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath { 728c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath first = _mm_move_ss(first,second); 729c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath first = _mm_shuffle_ps(first,second,0x93); 730c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath } 731c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath } 732c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath}; 733c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 734c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<int Offset> 735c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathstruct palign_impl<Offset,Packet4i> 736c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 737c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath static EIGEN_STRONG_INLINE void run(Packet4i& first, const Packet4i& second) 738c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath { 739c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath if (Offset==1) 740c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath { 741c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath first = _mm_castps_si128(_mm_move_ss(_mm_castsi128_ps(first),_mm_castsi128_ps(second))); 742c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath first = _mm_shuffle_epi32(first,0x39); 743c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath } 744c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath else if (Offset==2) 745c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath { 746c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath first = _mm_castps_si128(_mm_movehl_ps(_mm_castsi128_ps(first),_mm_castsi128_ps(first))); 747c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath first = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(first),_mm_castsi128_ps(second))); 748c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath } 749c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath else if (Offset==3) 750c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath { 751c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath first = _mm_castps_si128(_mm_move_ss(_mm_castsi128_ps(first),_mm_castsi128_ps(second))); 752c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath first = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(first),_mm_castsi128_ps(second),0x93)); 753c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath } 754c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath } 755c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath}; 756c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 757c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<int Offset> 758c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathstruct palign_impl<Offset,Packet2d> 759c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 760c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath static EIGEN_STRONG_INLINE void run(Packet2d& first, const Packet2d& second) 761c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath { 762c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath if (Offset==1) 763c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath { 764c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath first = _mm_castps_pd(_mm_movehl_ps(_mm_castpd_ps(first),_mm_castpd_ps(first))); 765c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath first = _mm_castps_pd(_mm_movelh_ps(_mm_castpd_ps(first),_mm_castpd_ps(second))); 766c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath } 767c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath } 768c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath}; 769c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#endif 770c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 7712b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao WangEIGEN_DEVICE_FUNC inline void 7722b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangptranspose(PacketBlock<Packet4f,4>& kernel) { 7732b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang _MM_TRANSPOSE4_PS(kernel.packet[0], kernel.packet[1], kernel.packet[2], kernel.packet[3]); 7742b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 7752b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 7762b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao WangEIGEN_DEVICE_FUNC inline void 7772b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangptranspose(PacketBlock<Packet2d,2>& kernel) { 7782b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m128d tmp = _mm_unpackhi_pd(kernel.packet[0], kernel.packet[1]); 7792b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang kernel.packet[0] = _mm_unpacklo_pd(kernel.packet[0], kernel.packet[1]); 7802b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang kernel.packet[1] = tmp; 7812b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 7822b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 7832b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao WangEIGEN_DEVICE_FUNC inline void 7842b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangptranspose(PacketBlock<Packet4i,4>& kernel) { 7852b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m128i T0 = _mm_unpacklo_epi32(kernel.packet[0], kernel.packet[1]); 7862b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m128i T1 = _mm_unpacklo_epi32(kernel.packet[2], kernel.packet[3]); 7872b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m128i T2 = _mm_unpackhi_epi32(kernel.packet[0], kernel.packet[1]); 7882b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m128i T3 = _mm_unpackhi_epi32(kernel.packet[2], kernel.packet[3]); 7892b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 7902b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang kernel.packet[0] = _mm_unpacklo_epi64(T0, T1); 7912b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang kernel.packet[1] = _mm_unpackhi_epi64(T0, T1); 7922b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang kernel.packet[2] = _mm_unpacklo_epi64(T2, T3); 7932b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang kernel.packet[3] = _mm_unpackhi_epi64(T2, T3); 7942b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 7952b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 7962b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4i pblend(const Selector<4>& ifPacket, const Packet4i& thenPacket, const Packet4i& elsePacket) { 7972b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang const __m128i zero = _mm_setzero_si128(); 7982b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang const __m128i select = _mm_set_epi32(ifPacket.select[3], ifPacket.select[2], ifPacket.select[1], ifPacket.select[0]); 7992b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m128i false_mask = _mm_cmpeq_epi32(select, zero); 8002b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifdef EIGEN_VECTORIZE_SSE4_1 8012b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return _mm_blendv_epi8(thenPacket, elsePacket, false_mask); 8022b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#else 8032b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return _mm_or_si128(_mm_andnot_si128(false_mask, thenPacket), _mm_and_si128(false_mask, elsePacket)); 8042b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 8052b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 8062b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4f pblend(const Selector<4>& ifPacket, const Packet4f& thenPacket, const Packet4f& elsePacket) { 8072b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang const __m128 zero = _mm_setzero_ps(); 8082b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang const __m128 select = _mm_set_ps(ifPacket.select[3], ifPacket.select[2], ifPacket.select[1], ifPacket.select[0]); 8092b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m128 false_mask = _mm_cmpeq_ps(select, zero); 8102b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifdef EIGEN_VECTORIZE_SSE4_1 8112b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return _mm_blendv_ps(thenPacket, elsePacket, false_mask); 8122b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#else 8132b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return _mm_or_ps(_mm_andnot_ps(false_mask, thenPacket), _mm_and_ps(false_mask, elsePacket)); 8142b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 8152b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 8162b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d pblend(const Selector<2>& ifPacket, const Packet2d& thenPacket, const Packet2d& elsePacket) { 8172b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang const __m128d zero = _mm_setzero_pd(); 8182b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang const __m128d select = _mm_set_pd(ifPacket.select[1], ifPacket.select[0]); 8192b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m128d false_mask = _mm_cmpeq_pd(select, zero); 8202b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifdef EIGEN_VECTORIZE_SSE4_1 8212b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return _mm_blendv_pd(thenPacket, elsePacket, false_mask); 8222b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#else 8232b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return _mm_or_pd(_mm_andnot_pd(false_mask, thenPacket), _mm_and_pd(false_mask, elsePacket)); 8242b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 8252b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 8262b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 8272b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4f pinsertfirst(const Packet4f& a, float b) 8282b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 8292b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifdef EIGEN_VECTORIZE_SSE4_1 8302b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return _mm_blend_ps(a,pset1<Packet4f>(b),1); 8312b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#else 8322b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return _mm_move_ss(a, _mm_load_ss(&b)); 8332b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 8342b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 8352b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 8362b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d pinsertfirst(const Packet2d& a, double b) 8372b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 8382b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifdef EIGEN_VECTORIZE_SSE4_1 8392b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return _mm_blend_pd(a,pset1<Packet2d>(b),1); 8402b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#else 8412b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return _mm_move_sd(a, _mm_load_sd(&b)); 8422b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 8432b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 8442b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 8452b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4f pinsertlast(const Packet4f& a, float b) 8462b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 8472b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifdef EIGEN_VECTORIZE_SSE4_1 8482b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return _mm_blend_ps(a,pset1<Packet4f>(b),(1<<3)); 8492b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#else 8502b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang const Packet4f mask = _mm_castsi128_ps(_mm_setr_epi32(0x0,0x0,0x0,0xFFFFFFFF)); 8512b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return _mm_or_ps(_mm_andnot_ps(mask, a), _mm_and_ps(mask, pset1<Packet4f>(b))); 8522b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 8532b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 8542b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 8552b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d pinsertlast(const Packet2d& a, double b) 8562b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 8572b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifdef EIGEN_VECTORIZE_SSE4_1 8582b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return _mm_blend_pd(a,pset1<Packet2d>(b),(1<<1)); 8592b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#else 8602b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang const Packet2d mask = _mm_castsi128_pd(_mm_setr_epi32(0x0,0x0,0xFFFFFFFF,0xFFFFFFFF)); 8612b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return _mm_or_pd(_mm_andnot_pd(mask, a), _mm_and_pd(mask, pset1<Packet2d>(b))); 8622b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 8632b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 8642b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 8652b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// Scalar path for pmadd with FMA to ensure consistency with vectorized path. 8662b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifdef __FMA__ 8672b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE float pmadd(const float& a, const float& b, const float& c) { 8682b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return ::fmaf(a,b,c); 8692b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 8702b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE double pmadd(const double& a, const double& b, const double& c) { 8712b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return ::fma(a,b,c); 8722b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 8732b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 8742b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 875c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} // end namespace internal 876c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 877c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} // end namespace Eigen 878c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 879c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#endif // EIGEN_PACKET_MATH_SSE_H 880