12b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// This file is part of Eigen, a lightweight C++ template library 22b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// for linear algebra. 32b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// 42b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// Copyright (C) 2014 Benoit Steiner (benoit.steiner.goog@gmail.com) 52b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// 62b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// This Source Code Form is subject to the terms of the Mozilla 72b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// Public License v. 2.0. If a copy of the MPL was not distributed 82b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 92b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 102b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifndef EIGEN_COMPLEX_AVX_H 112b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#define EIGEN_COMPLEX_AVX_H 122b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 132b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangnamespace Eigen { 142b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 152b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangnamespace internal { 162b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 172b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang//---------- float ---------- 182b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangstruct Packet4cf 192b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 202b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_STRONG_INLINE Packet4cf() {} 212b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_STRONG_INLINE explicit Packet4cf(const __m256& a) : v(a) {} 222b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m256 v; 232b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang}; 242b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 252b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> struct packet_traits<std::complex<float> > : default_packet_traits 262b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 272b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang typedef Packet4cf type; 282b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang typedef Packet2cf half; 292b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang enum { 302b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Vectorizable = 1, 312b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang AlignedOnScalar = 1, 322b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang size = 4, 332b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasHalfPacket = 1, 342b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 352b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasAdd = 1, 362b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasSub = 1, 372b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasMul = 1, 382b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasDiv = 1, 392b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasNegate = 1, 402b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasAbs = 0, 412b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasAbs2 = 0, 422b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasMin = 0, 432b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasMax = 0, 442b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasSetLinear = 0 452b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang }; 462b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang}; 472b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 482b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> struct unpacket_traits<Packet4cf> { typedef std::complex<float> type; enum {size=4, alignment=Aligned32}; typedef Packet2cf half; }; 492b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 502b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4cf padd<Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_add_ps(a.v,b.v)); } 512b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4cf psub<Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_sub_ps(a.v,b.v)); } 522b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4cf pnegate(const Packet4cf& a) 532b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 542b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return Packet4cf(pnegate(a.v)); 552b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 562b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4cf pconj(const Packet4cf& a) 572b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 582b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang const __m256 mask = _mm256_castsi256_ps(_mm256_setr_epi32(0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000)); 592b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return Packet4cf(_mm256_xor_ps(a.v,mask)); 602b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 612b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 622b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4cf pmul<Packet4cf>(const Packet4cf& a, const Packet4cf& b) 632b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 642b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m256 tmp1 = _mm256_mul_ps(_mm256_moveldup_ps(a.v), b.v); 652b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m256 tmp2 = _mm256_mul_ps(_mm256_movehdup_ps(a.v), _mm256_permute_ps(b.v, _MM_SHUFFLE(2,3,0,1))); 662b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m256 result = _mm256_addsub_ps(tmp1, tmp2); 672b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return Packet4cf(result); 682b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 692b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 702b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4cf pand <Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_and_ps(a.v,b.v)); } 712b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4cf por <Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_or_ps(a.v,b.v)); } 722b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4cf pxor <Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_xor_ps(a.v,b.v)); } 732b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4cf pandnot<Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_andnot_ps(a.v,b.v)); } 742b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 752b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4cf pload <Packet4cf>(const std::complex<float>* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet4cf(pload<Packet8f>(&numext::real_ref(*from))); } 762b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4cf ploadu<Packet4cf>(const std::complex<float>* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet4cf(ploadu<Packet8f>(&numext::real_ref(*from))); } 772b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 782b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 792b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4cf pset1<Packet4cf>(const std::complex<float>& from) 802b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 812b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return Packet4cf(_mm256_castpd_ps(_mm256_broadcast_sd((const double*)(const void*)&from))); 822b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 832b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 842b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4cf ploaddup<Packet4cf>(const std::complex<float>* from) 852b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 862b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang // FIXME The following might be optimized using _mm256_movedup_pd 872b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Packet2cf a = ploaddup<Packet2cf>(from); 882b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Packet2cf b = ploaddup<Packet2cf>(from+1); 892b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return Packet4cf(_mm256_insertf128_ps(_mm256_castps128_ps256(a.v), b.v, 1)); 902b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 912b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 922b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE void pstore <std::complex<float> >(std::complex<float>* to, const Packet4cf& from) { EIGEN_DEBUG_ALIGNED_STORE pstore(&numext::real_ref(*to), from.v); } 932b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE void pstoreu<std::complex<float> >(std::complex<float>* to, const Packet4cf& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu(&numext::real_ref(*to), from.v); } 942b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 952b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_DEVICE_FUNC inline Packet4cf pgather<std::complex<float>, Packet4cf>(const std::complex<float>* from, Index stride) 962b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 972b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return Packet4cf(_mm256_set_ps(std::imag(from[3*stride]), std::real(from[3*stride]), 982b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang std::imag(from[2*stride]), std::real(from[2*stride]), 992b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang std::imag(from[1*stride]), std::real(from[1*stride]), 1002b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang std::imag(from[0*stride]), std::real(from[0*stride]))); 1012b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 1022b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 1032b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_DEVICE_FUNC inline void pscatter<std::complex<float>, Packet4cf>(std::complex<float>* to, const Packet4cf& from, Index stride) 1042b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 1052b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m128 low = _mm256_extractf128_ps(from.v, 0); 1062b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang to[stride*0] = std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(low, low, 0)), 1072b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang _mm_cvtss_f32(_mm_shuffle_ps(low, low, 1))); 1082b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang to[stride*1] = std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(low, low, 2)), 1092b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang _mm_cvtss_f32(_mm_shuffle_ps(low, low, 3))); 1102b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 1112b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m128 high = _mm256_extractf128_ps(from.v, 1); 1122b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang to[stride*2] = std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(high, high, 0)), 1132b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang _mm_cvtss_f32(_mm_shuffle_ps(high, high, 1))); 1142b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang to[stride*3] = std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(high, high, 2)), 1152b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang _mm_cvtss_f32(_mm_shuffle_ps(high, high, 3))); 1162b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 1172b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 1182b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 1192b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE std::complex<float> pfirst<Packet4cf>(const Packet4cf& a) 1202b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 1212b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return pfirst(Packet2cf(_mm256_castps256_ps128(a.v))); 1222b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 1232b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 1242b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4cf preverse(const Packet4cf& a) { 1252b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m128 low = _mm256_extractf128_ps(a.v, 0); 1262b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m128 high = _mm256_extractf128_ps(a.v, 1); 1272b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m128d lowd = _mm_castps_pd(low); 1282b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m128d highd = _mm_castps_pd(high); 1292b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang low = _mm_castpd_ps(_mm_shuffle_pd(lowd,lowd,0x1)); 1302b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang high = _mm_castpd_ps(_mm_shuffle_pd(highd,highd,0x1)); 1312b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m256 result = _mm256_setzero_ps(); 1322b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang result = _mm256_insertf128_ps(result, low, 1); 1332b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang result = _mm256_insertf128_ps(result, high, 0); 1342b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return Packet4cf(result); 1352b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 1362b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 1372b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE std::complex<float> predux<Packet4cf>(const Packet4cf& a) 1382b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 1392b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return predux(padd(Packet2cf(_mm256_extractf128_ps(a.v,0)), 1402b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Packet2cf(_mm256_extractf128_ps(a.v,1)))); 1412b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 1422b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 1432b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4cf preduxp<Packet4cf>(const Packet4cf* vecs) 1442b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 1452b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Packet8f t0 = _mm256_shuffle_ps(vecs[0].v, vecs[0].v, _MM_SHUFFLE(3, 1, 2 ,0)); 1462b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Packet8f t1 = _mm256_shuffle_ps(vecs[1].v, vecs[1].v, _MM_SHUFFLE(3, 1, 2 ,0)); 1472b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang t0 = _mm256_hadd_ps(t0,t1); 1482b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Packet8f t2 = _mm256_shuffle_ps(vecs[2].v, vecs[2].v, _MM_SHUFFLE(3, 1, 2 ,0)); 1492b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Packet8f t3 = _mm256_shuffle_ps(vecs[3].v, vecs[3].v, _MM_SHUFFLE(3, 1, 2 ,0)); 1502b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang t2 = _mm256_hadd_ps(t2,t3); 1512b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 1522b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang t1 = _mm256_permute2f128_ps(t0,t2, 0 + (2<<4)); 1532b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang t3 = _mm256_permute2f128_ps(t0,t2, 1 + (3<<4)); 1542b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 1552b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return Packet4cf(_mm256_add_ps(t1,t3)); 1562b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 1572b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 1582b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE std::complex<float> predux_mul<Packet4cf>(const Packet4cf& a) 1592b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 1602b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return predux_mul(pmul(Packet2cf(_mm256_extractf128_ps(a.v, 0)), 1612b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Packet2cf(_mm256_extractf128_ps(a.v, 1)))); 1622b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 1632b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 1642b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<int Offset> 1652b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangstruct palign_impl<Offset,Packet4cf> 1662b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 1672b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang static EIGEN_STRONG_INLINE void run(Packet4cf& first, const Packet4cf& second) 1682b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang { 1692b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang if (Offset==0) return; 1702b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang palign_impl<Offset*2,Packet8f>::run(first.v, second.v); 1712b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang } 1722b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang}; 1732b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 1742b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> struct conj_helper<Packet4cf, Packet4cf, false,true> 1752b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 1762b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_STRONG_INLINE Packet4cf pmadd(const Packet4cf& x, const Packet4cf& y, const Packet4cf& c) const 1772b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang { return padd(pmul(x,y),c); } 1782b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 1792b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_STRONG_INLINE Packet4cf pmul(const Packet4cf& a, const Packet4cf& b) const 1802b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang { 1812b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return internal::pmul(a, pconj(b)); 1822b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang } 1832b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang}; 1842b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 1852b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> struct conj_helper<Packet4cf, Packet4cf, true,false> 1862b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 1872b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_STRONG_INLINE Packet4cf pmadd(const Packet4cf& x, const Packet4cf& y, const Packet4cf& c) const 1882b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang { return padd(pmul(x,y),c); } 1892b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 1902b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_STRONG_INLINE Packet4cf pmul(const Packet4cf& a, const Packet4cf& b) const 1912b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang { 1922b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return internal::pmul(pconj(a), b); 1932b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang } 1942b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang}; 1952b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 1962b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> struct conj_helper<Packet4cf, Packet4cf, true,true> 1972b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 1982b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_STRONG_INLINE Packet4cf pmadd(const Packet4cf& x, const Packet4cf& y, const Packet4cf& c) const 1992b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang { return padd(pmul(x,y),c); } 2002b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 2012b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_STRONG_INLINE Packet4cf pmul(const Packet4cf& a, const Packet4cf& b) const 2022b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang { 2032b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return pconj(internal::pmul(a, b)); 2042b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang } 2052b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang}; 2062b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 2072b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> struct conj_helper<Packet8f, Packet4cf, false,false> 2082b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 2092b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_STRONG_INLINE Packet4cf pmadd(const Packet8f& x, const Packet4cf& y, const Packet4cf& c) const 2102b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang { return padd(c, pmul(x,y)); } 2112b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 2122b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_STRONG_INLINE Packet4cf pmul(const Packet8f& x, const Packet4cf& y) const 2132b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang { return Packet4cf(Eigen::internal::pmul(x, y.v)); } 2142b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang}; 2152b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 2162b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> struct conj_helper<Packet4cf, Packet8f, false,false> 2172b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 2182b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_STRONG_INLINE Packet4cf pmadd(const Packet4cf& x, const Packet8f& y, const Packet4cf& c) const 2192b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang { return padd(c, pmul(x,y)); } 2202b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 2212b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_STRONG_INLINE Packet4cf pmul(const Packet4cf& x, const Packet8f& y) const 2222b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang { return Packet4cf(Eigen::internal::pmul(x.v, y)); } 2232b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang}; 2242b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 2252b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4cf pdiv<Packet4cf>(const Packet4cf& a, const Packet4cf& b) 2262b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 2272b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Packet4cf num = pmul(a, pconj(b)); 2282b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m256 tmp = _mm256_mul_ps(b.v, b.v); 2292b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m256 tmp2 = _mm256_shuffle_ps(tmp,tmp,0xB1); 2302b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m256 denom = _mm256_add_ps(tmp, tmp2); 2312b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return Packet4cf(_mm256_div_ps(num.v, denom)); 2322b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 2332b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 2342b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4cf pcplxflip<Packet4cf>(const Packet4cf& x) 2352b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 2362b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return Packet4cf(_mm256_shuffle_ps(x.v, x.v, _MM_SHUFFLE(2, 3, 0 ,1))); 2372b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 2382b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 2392b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang//---------- double ---------- 2402b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangstruct Packet2cd 2412b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 2422b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_STRONG_INLINE Packet2cd() {} 2432b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_STRONG_INLINE explicit Packet2cd(const __m256d& a) : v(a) {} 2442b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m256d v; 2452b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang}; 2462b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 2472b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> struct packet_traits<std::complex<double> > : default_packet_traits 2482b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 2492b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang typedef Packet2cd type; 2502b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang typedef Packet1cd half; 2512b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang enum { 2522b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Vectorizable = 1, 2532b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang AlignedOnScalar = 0, 2542b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang size = 2, 2552b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasHalfPacket = 1, 2562b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 2572b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasAdd = 1, 2582b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasSub = 1, 2592b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasMul = 1, 2602b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasDiv = 1, 2612b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasNegate = 1, 2622b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasAbs = 0, 2632b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasAbs2 = 0, 2642b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasMin = 0, 2652b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasMax = 0, 2662b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasSetLinear = 0 2672b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang }; 2682b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang}; 2692b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 2702b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> struct unpacket_traits<Packet2cd> { typedef std::complex<double> type; enum {size=2, alignment=Aligned32}; typedef Packet1cd half; }; 2712b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 2722b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2cd padd<Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_add_pd(a.v,b.v)); } 2732b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2cd psub<Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_sub_pd(a.v,b.v)); } 2742b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2cd pnegate(const Packet2cd& a) { return Packet2cd(pnegate(a.v)); } 2752b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2cd pconj(const Packet2cd& a) 2762b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 2772b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang const __m256d mask = _mm256_castsi256_pd(_mm256_set_epi32(0x80000000,0x0,0x0,0x0,0x80000000,0x0,0x0,0x0)); 2782b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return Packet2cd(_mm256_xor_pd(a.v,mask)); 2792b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 2802b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 2812b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2cd pmul<Packet2cd>(const Packet2cd& a, const Packet2cd& b) 2822b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 2832b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m256d tmp1 = _mm256_shuffle_pd(a.v,a.v,0x0); 2842b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m256d even = _mm256_mul_pd(tmp1, b.v); 2852b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m256d tmp2 = _mm256_shuffle_pd(a.v,a.v,0xF); 2862b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m256d tmp3 = _mm256_shuffle_pd(b.v,b.v,0x5); 2872b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m256d odd = _mm256_mul_pd(tmp2, tmp3); 2882b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return Packet2cd(_mm256_addsub_pd(even, odd)); 2892b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 2902b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 2912b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2cd pand <Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_and_pd(a.v,b.v)); } 2922b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2cd por <Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_or_pd(a.v,b.v)); } 2932b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2cd pxor <Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_xor_pd(a.v,b.v)); } 2942b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2cd pandnot<Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_andnot_pd(a.v,b.v)); } 2952b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 2962b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2cd pload <Packet2cd>(const std::complex<double>* from) 2972b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ EIGEN_DEBUG_ALIGNED_LOAD return Packet2cd(pload<Packet4d>((const double*)from)); } 2982b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2cd ploadu<Packet2cd>(const std::complex<double>* from) 2992b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ EIGEN_DEBUG_UNALIGNED_LOAD return Packet2cd(ploadu<Packet4d>((const double*)from)); } 3002b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 3012b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2cd pset1<Packet2cd>(const std::complex<double>& from) 3022b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 3032b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang // in case casting to a __m128d* is really not safe, then we can still fallback to this version: (much slower though) 3042b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// return Packet2cd(_mm256_loadu2_m128d((const double*)&from,(const double*)&from)); 3052b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return Packet2cd(_mm256_broadcast_pd((const __m128d*)(const void*)&from)); 3062b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 3072b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 3082b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2cd ploaddup<Packet2cd>(const std::complex<double>* from) { return pset1<Packet2cd>(*from); } 3092b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 3102b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE void pstore <std::complex<double> >(std::complex<double> * to, const Packet2cd& from) { EIGEN_DEBUG_ALIGNED_STORE pstore((double*)to, from.v); } 3112b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE void pstoreu<std::complex<double> >(std::complex<double> * to, const Packet2cd& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, from.v); } 3122b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 3132b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_DEVICE_FUNC inline Packet2cd pgather<std::complex<double>, Packet2cd>(const std::complex<double>* from, Index stride) 3142b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 3152b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return Packet2cd(_mm256_set_pd(std::imag(from[1*stride]), std::real(from[1*stride]), 3162b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang std::imag(from[0*stride]), std::real(from[0*stride]))); 3172b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 3182b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 3192b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_DEVICE_FUNC inline void pscatter<std::complex<double>, Packet2cd>(std::complex<double>* to, const Packet2cd& from, Index stride) 3202b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 3212b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m128d low = _mm256_extractf128_pd(from.v, 0); 3222b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang to[stride*0] = std::complex<double>(_mm_cvtsd_f64(low), _mm_cvtsd_f64(_mm_shuffle_pd(low, low, 1))); 3232b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m128d high = _mm256_extractf128_pd(from.v, 1); 3242b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang to[stride*1] = std::complex<double>(_mm_cvtsd_f64(high), _mm_cvtsd_f64(_mm_shuffle_pd(high, high, 1))); 3252b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 3262b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 3272b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE std::complex<double> pfirst<Packet2cd>(const Packet2cd& a) 3282b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 3292b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m128d low = _mm256_extractf128_pd(a.v, 0); 3302b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_ALIGN16 double res[2]; 3312b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang _mm_store_pd(res, low); 3322b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return std::complex<double>(res[0],res[1]); 3332b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 3342b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 3352b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2cd preverse(const Packet2cd& a) { 3362b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m256d result = _mm256_permute2f128_pd(a.v, a.v, 1); 3372b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return Packet2cd(result); 3382b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 3392b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 3402b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE std::complex<double> predux<Packet2cd>(const Packet2cd& a) 3412b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 3422b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return predux(padd(Packet1cd(_mm256_extractf128_pd(a.v,0)), 3432b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Packet1cd(_mm256_extractf128_pd(a.v,1)))); 3442b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 3452b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 3462b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2cd preduxp<Packet2cd>(const Packet2cd* vecs) 3472b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 3482b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Packet4d t0 = _mm256_permute2f128_pd(vecs[0].v,vecs[1].v, 0 + (2<<4)); 3492b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Packet4d t1 = _mm256_permute2f128_pd(vecs[0].v,vecs[1].v, 1 + (3<<4)); 3502b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 3512b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return Packet2cd(_mm256_add_pd(t0,t1)); 3522b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 3532b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 3542b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE std::complex<double> predux_mul<Packet2cd>(const Packet2cd& a) 3552b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 3562b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return predux(pmul(Packet1cd(_mm256_extractf128_pd(a.v,0)), 3572b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Packet1cd(_mm256_extractf128_pd(a.v,1)))); 3582b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 3592b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 3602b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<int Offset> 3612b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangstruct palign_impl<Offset,Packet2cd> 3622b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 3632b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang static EIGEN_STRONG_INLINE void run(Packet2cd& first, const Packet2cd& second) 3642b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang { 3652b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang if (Offset==0) return; 3662b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang palign_impl<Offset*2,Packet4d>::run(first.v, second.v); 3672b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang } 3682b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang}; 3692b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 3702b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> struct conj_helper<Packet2cd, Packet2cd, false,true> 3712b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 3722b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_STRONG_INLINE Packet2cd pmadd(const Packet2cd& x, const Packet2cd& y, const Packet2cd& c) const 3732b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang { return padd(pmul(x,y),c); } 3742b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 3752b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_STRONG_INLINE Packet2cd pmul(const Packet2cd& a, const Packet2cd& b) const 3762b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang { 3772b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return internal::pmul(a, pconj(b)); 3782b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang } 3792b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang}; 3802b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 3812b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> struct conj_helper<Packet2cd, Packet2cd, true,false> 3822b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 3832b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_STRONG_INLINE Packet2cd pmadd(const Packet2cd& x, const Packet2cd& y, const Packet2cd& c) const 3842b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang { return padd(pmul(x,y),c); } 3852b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 3862b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_STRONG_INLINE Packet2cd pmul(const Packet2cd& a, const Packet2cd& b) const 3872b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang { 3882b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return internal::pmul(pconj(a), b); 3892b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang } 3902b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang}; 3912b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 3922b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> struct conj_helper<Packet2cd, Packet2cd, true,true> 3932b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 3942b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_STRONG_INLINE Packet2cd pmadd(const Packet2cd& x, const Packet2cd& y, const Packet2cd& c) const 3952b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang { return padd(pmul(x,y),c); } 3962b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 3972b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_STRONG_INLINE Packet2cd pmul(const Packet2cd& a, const Packet2cd& b) const 3982b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang { 3992b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return pconj(internal::pmul(a, b)); 4002b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang } 4012b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang}; 4022b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 4032b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> struct conj_helper<Packet4d, Packet2cd, false,false> 4042b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 4052b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_STRONG_INLINE Packet2cd pmadd(const Packet4d& x, const Packet2cd& y, const Packet2cd& c) const 4062b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang { return padd(c, pmul(x,y)); } 4072b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 4082b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_STRONG_INLINE Packet2cd pmul(const Packet4d& x, const Packet2cd& y) const 4092b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang { return Packet2cd(Eigen::internal::pmul(x, y.v)); } 4102b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang}; 4112b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 4122b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> struct conj_helper<Packet2cd, Packet4d, false,false> 4132b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 4142b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_STRONG_INLINE Packet2cd pmadd(const Packet2cd& x, const Packet4d& y, const Packet2cd& c) const 4152b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang { return padd(c, pmul(x,y)); } 4162b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 4172b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_STRONG_INLINE Packet2cd pmul(const Packet2cd& x, const Packet4d& y) const 4182b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang { return Packet2cd(Eigen::internal::pmul(x.v, y)); } 4192b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang}; 4202b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 4212b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2cd pdiv<Packet2cd>(const Packet2cd& a, const Packet2cd& b) 4222b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 4232b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Packet2cd num = pmul(a, pconj(b)); 4242b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m256d tmp = _mm256_mul_pd(b.v, b.v); 4252b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m256d denom = _mm256_hadd_pd(tmp, tmp); 4262b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return Packet2cd(_mm256_div_pd(num.v, denom)); 4272b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 4282b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 4292b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2cd pcplxflip<Packet2cd>(const Packet2cd& x) 4302b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 4312b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return Packet2cd(_mm256_shuffle_pd(x.v, x.v, 0x5)); 4322b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 4332b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 4342b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao WangEIGEN_DEVICE_FUNC inline void 4352b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangptranspose(PacketBlock<Packet4cf,4>& kernel) { 4362b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m256d P0 = _mm256_castps_pd(kernel.packet[0].v); 4372b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m256d P1 = _mm256_castps_pd(kernel.packet[1].v); 4382b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m256d P2 = _mm256_castps_pd(kernel.packet[2].v); 4392b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m256d P3 = _mm256_castps_pd(kernel.packet[3].v); 4402b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 4412b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m256d T0 = _mm256_shuffle_pd(P0, P1, 15); 4422b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m256d T1 = _mm256_shuffle_pd(P0, P1, 0); 4432b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m256d T2 = _mm256_shuffle_pd(P2, P3, 15); 4442b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m256d T3 = _mm256_shuffle_pd(P2, P3, 0); 4452b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 4462b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang kernel.packet[1].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T0, T2, 32)); 4472b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang kernel.packet[3].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T0, T2, 49)); 4482b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang kernel.packet[0].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T1, T3, 32)); 4492b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang kernel.packet[2].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T1, T3, 49)); 4502b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 4512b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 4522b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao WangEIGEN_DEVICE_FUNC inline void 4532b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangptranspose(PacketBlock<Packet2cd,2>& kernel) { 4542b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang __m256d tmp = _mm256_permute2f128_pd(kernel.packet[0].v, kernel.packet[1].v, 0+(2<<4)); 4552b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang kernel.packet[1].v = _mm256_permute2f128_pd(kernel.packet[0].v, kernel.packet[1].v, 1+(3<<4)); 4562b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang kernel.packet[0].v = tmp; 4572b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 4582b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 4592b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4cf pinsertfirst(const Packet4cf& a, std::complex<float> b) 4602b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 4612b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return Packet4cf(_mm256_blend_ps(a.v,pset1<Packet4cf>(b).v,1|2)); 4622b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 4632b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 4642b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2cd pinsertfirst(const Packet2cd& a, std::complex<double> b) 4652b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 4662b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return Packet2cd(_mm256_blend_pd(a.v,pset1<Packet2cd>(b).v,1|2)); 4672b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 4682b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 4692b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4cf pinsertlast(const Packet4cf& a, std::complex<float> b) 4702b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 4712b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return Packet4cf(_mm256_blend_ps(a.v,pset1<Packet4cf>(b).v,(1<<7)|(1<<6))); 4722b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 4732b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 4742b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2cd pinsertlast(const Packet2cd& a, std::complex<double> b) 4752b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 4762b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return Packet2cd(_mm256_blend_pd(a.v,pset1<Packet2cd>(b).v,(1<<3)|(1<<2))); 4772b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 4782b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 4792b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} // end namespace internal 4802b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 4812b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} // end namespace Eigen 4822b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 4832b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif // EIGEN_COMPLEX_AVX_H 484