1c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// This file is part of Eigen, a lightweight C++ template library 2c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// for linear algebra. 3c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// 4c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// Copyright (C) 2008-2009 Gael Guennebaud <gael.guennebaud@inria.fr> 52b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// Copyright (C) 2010 Konstantinos Margaritis <markos@freevec.org> 6c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// Heavily based on Gael's SSE version. 7c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// 8c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// This Source Code Form is subject to the terms of the Mozilla 9c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// Public License v. 2.0. If a copy of the MPL was not distributed 10c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 11c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 12c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#ifndef EIGEN_PACKET_MATH_NEON_H 13c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#define EIGEN_PACKET_MATH_NEON_H 14c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 15c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathnamespace Eigen { 16c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 17c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathnamespace internal { 18c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 19c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 20c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8 21c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#endif 22c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 232b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD 242b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD 252b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 262b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 272b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_CJMADD 282b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#define EIGEN_HAS_SINGLE_INSTRUCTION_CJMADD 292b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 302b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 31c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 322b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#if EIGEN_ARCH_ARM64 332b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32 342b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#else 352b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 16 362b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 37c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#endif 38c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 392b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtypedef float32x2_t Packet2f; 40c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtypedef float32x4_t Packet4f; 41c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtypedef int32x4_t Packet4i; 422b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtypedef int32x2_t Packet2i; 43c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtypedef uint32x4_t Packet4ui; 44c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 45c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \ 46c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath const Packet4f p4f_##NAME = pset1<Packet4f>(X) 47c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 48c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \ 492b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang const Packet4f p4f_##NAME = vreinterpretq_f32_u32(pset1<int32_t>(X)) 50c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 51c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \ 52c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath const Packet4i p4i_##NAME = pset1<Packet4i>(X) 53c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 54eda03298de395cf6217486971e6529f92da8da79Miao Wang#if EIGEN_ARCH_ARM64 55eda03298de395cf6217486971e6529f92da8da79Miao Wang // __builtin_prefetch tends to do nothing on ARM64 compilers because the 56eda03298de395cf6217486971e6529f92da8da79Miao Wang // prefetch instructions there are too detailed for __builtin_prefetch to map 57eda03298de395cf6217486971e6529f92da8da79Miao Wang // meaningfully to them. 58eda03298de395cf6217486971e6529f92da8da79Miao Wang #define EIGEN_ARM_PREFETCH(ADDR) __asm__ __volatile__("prfm pldl1keep, [%[addr]]\n" ::[addr] "r"(ADDR) : ); 59eda03298de395cf6217486971e6529f92da8da79Miao Wang#elif EIGEN_HAS_BUILTIN(__builtin_prefetch) || EIGEN_COMP_GNUC 60615d816d068b4d0f5e8df601930b5f160bf7eda1Tim Murray #define EIGEN_ARM_PREFETCH(ADDR) __builtin_prefetch(ADDR); 61615d816d068b4d0f5e8df601930b5f160bf7eda1Tim Murray#elif defined __pld 62615d816d068b4d0f5e8df601930b5f160bf7eda1Tim Murray #define EIGEN_ARM_PREFETCH(ADDR) __pld(ADDR) 63eda03298de395cf6217486971e6529f92da8da79Miao Wang#elif EIGEN_ARCH_ARM32 64eda03298de395cf6217486971e6529f92da8da79Miao Wang #define EIGEN_ARM_PREFETCH(ADDR) __asm__ __volatile__ ("pld [%[addr]]\n" :: [addr] "r" (ADDR) : ); 65615d816d068b4d0f5e8df601930b5f160bf7eda1Tim Murray#else 66615d816d068b4d0f5e8df601930b5f160bf7eda1Tim Murray // by default no explicit prefetching 67615d816d068b4d0f5e8df601930b5f160bf7eda1Tim Murray #define EIGEN_ARM_PREFETCH(ADDR) 68c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#endif 69c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 70c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> struct packet_traits<float> : default_packet_traits 71c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 72c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath typedef Packet4f type; 732b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang typedef Packet4f half; // Packet2f intrinsics not implemented yet 74c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath enum { 75c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath Vectorizable = 1, 76c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath AlignedOnScalar = 1, 77c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath size = 4, 782b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasHalfPacket=0, // Packet2f intrinsics not implemented yet 79c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 80c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath HasDiv = 1, 81c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // FIXME check the Has* 82c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath HasSin = 0, 83c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath HasCos = 0, 84c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath HasLog = 0, 852b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasExp = 1, 86c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath HasSqrt = 0 87c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath }; 88c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath}; 892b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> struct packet_traits<int32_t> : default_packet_traits 90c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 91c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath typedef Packet4i type; 922b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang typedef Packet4i half; // Packet2i intrinsics not implemented yet 93c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath enum { 94c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath Vectorizable = 1, 95c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath AlignedOnScalar = 1, 962b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang size=4, 972b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasHalfPacket=0 // Packet2i intrinsics not implemented yet 98c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // FIXME check the Has* 99c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath }; 100c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath}; 101c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 1022b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#if EIGEN_GNUC_AT_MOST(4,4) && !EIGEN_COMP_LLVM 103c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// workaround gcc 4.2, 4.3 and 4.4 compilatin issue 104c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan KamathEIGEN_STRONG_INLINE float32x4_t vld1q_f32(const float* x) { return ::vld1q_f32((const float32_t*)x); } 105c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan KamathEIGEN_STRONG_INLINE float32x2_t vld1_f32 (const float* x) { return ::vld1_f32 ((const float32_t*)x); } 1062b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao WangEIGEN_STRONG_INLINE float32x2_t vld1_dup_f32 (const float* x) { return ::vld1_dup_f32 ((const float32_t*)x); } 107c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan KamathEIGEN_STRONG_INLINE void vst1q_f32(float* to, float32x4_t from) { ::vst1q_f32((float32_t*)to,from); } 108c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan KamathEIGEN_STRONG_INLINE void vst1_f32 (float* to, float32x2_t from) { ::vst1_f32 ((float32_t*)to,from); } 109c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#endif 110c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 1112b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> struct unpacket_traits<Packet4f> { typedef float type; enum {size=4, alignment=Aligned16}; typedef Packet4f half; }; 1122b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> struct unpacket_traits<Packet4i> { typedef int32_t type; enum {size=4, alignment=Aligned16}; typedef Packet4i half; }; 113c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 114c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) { return vdupq_n_f32(from); } 1152b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int32_t& from) { return vdupq_n_s32(from); } 116c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 1172b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(const float& a) 118c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 119eda03298de395cf6217486971e6529f92da8da79Miao Wang const float f[] = {0, 1, 2, 3}; 1202b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Packet4f countdown = vld1q_f32(f); 121c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return vaddq_f32(pset1<Packet4f>(a), countdown); 122c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 1232b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(const int32_t& a) 124c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 1252b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang const int32_t i[] = {0, 1, 2, 3}; 1262b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Packet4i countdown = vld1q_s32(i); 127c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return vaddq_s32(pset1<Packet4i>(a), countdown); 128c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 129c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 130c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) { return vaddq_f32(a,b); } 131c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(const Packet4i& a, const Packet4i& b) { return vaddq_s32(a,b); } 132c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 133c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) { return vsubq_f32(a,b); } 134c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const Packet4i& b) { return vsubq_s32(a,b); } 135c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 136c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) { return vnegq_f32(a); } 137c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) { return vnegq_s32(a); } 138c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 1397faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandeztemplate<> EIGEN_STRONG_INLINE Packet4f pconj(const Packet4f& a) { return a; } 1407faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandeztemplate<> EIGEN_STRONG_INLINE Packet4i pconj(const Packet4i& a) { return a; } 1417faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez 142c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) { return vmulq_f32(a,b); } 143c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b) { return vmulq_s32(a,b); } 144c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 145c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b) 146c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 1472b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#if EIGEN_ARCH_ARM64 1482b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return vdivq_f32(a,b); 1492b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#else 150c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath Packet4f inv, restep, div; 151c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 152c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // NEON does not offer a divide instruction, we have to do a reciprocal approximation 153c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // However NEON in contrast to other SIMD engines (AltiVec/SSE), offers 154c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // a reciprocal estimate AND a reciprocal step -which saves a few instructions 155c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // vrecpeq_f32() returns an estimate to 1/b, which we will finetune with 156c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // Newton-Raphson and vrecpsq_f32() 157c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath inv = vrecpeq_f32(b); 158c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 159c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // This returns a differential, by which we will have to multiply inv to get a better 160c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // approximation of 1/b. 161c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath restep = vrecpsq_f32(b, inv); 162c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath inv = vmulq_f32(restep, inv); 163c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 164c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // Finally, multiply a by 1/b and get the wanted result of the division. 165c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath div = vmulq_f32(a, inv); 166c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 167c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return div; 1682b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 169c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 1702b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 171c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(const Packet4i& /*a*/, const Packet4i& /*b*/) 172c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ eigen_assert(false && "packet integer division are not supported by NEON"); 173c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return pset1<Packet4i>(0); 174c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 175c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 1762b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// Clang/ARM wrongly advertises __ARM_FEATURE_FMA even when it's not available, 1772b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// then implements a slow software scalar fallback calling fmaf()! 1782b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// Filed LLVM bug: 1792b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// https://llvm.org/bugs/show_bug.cgi?id=27216 1802b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#if (defined __ARM_FEATURE_FMA) && !(EIGEN_COMP_CLANG && EIGEN_ARCH_ARM) 1812b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// See bug 936. 1822b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// FMA is available on VFPv4 i.e. when compiling with -mfpu=neon-vfpv4. 1832b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// FMA is a true fused multiply-add i.e. only 1 rounding at the end, no intermediate rounding. 1842b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// MLA is not fused i.e. does 2 roundings. 1852b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// In addition to giving better accuracy, FMA also gives better performance here on a Krait (Nexus 4): 1862b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// MLA: 10 GFlop/s ; FMA: 12 GFlops/s. 1872b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) { return vfmaq_f32(c,a,b); } 1882b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#else 1892b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) { 1902b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#if EIGEN_COMP_CLANG && EIGEN_ARCH_ARM 1912b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang // Clang/ARM will replace VMLA by VMUL+VADD at least for some values of -mcpu, 1922b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang // at least -mcpu=cortex-a8 and -mcpu=cortex-a7. Since the former is the default on 1932b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang // -march=armv7-a, that is a very common case. 1942b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang // See e.g. this thread: 1952b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang // http://lists.llvm.org/pipermail/llvm-dev/2013-December/068806.html 1962b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang // Filed LLVM bug: 1972b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang // https://llvm.org/bugs/show_bug.cgi?id=27219 1982b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Packet4f r = c; 1992b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang asm volatile( 2002b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang "vmla.f32 %q[r], %q[a], %q[b]" 2012b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang : [r] "+w" (r) 2022b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang : [a] "w" (a), 2032b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang [b] "w" (b) 2042b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang : ); 2052b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return r; 2062b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#else 2072b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return vmlaq_f32(c,a,b); 2082b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 2092b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 2102b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 2112b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 2122b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// No FMA instruction for int, so use MLA unconditionally. 213c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) { return vmlaq_s32(c,a,b); } 214c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 215c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) { return vminq_f32(a,b); } 216c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b) { return vminq_s32(a,b); } 217c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 218c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) { return vmaxq_f32(a,b); } 219c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b) { return vmaxq_s32(a,b); } 220c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 221c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// Logical Operations are not supported for float, so we have to reinterpret casts using NEON intrinsics 222c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b) 223c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 224c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b))); 225c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 226c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(const Packet4i& a, const Packet4i& b) { return vandq_s32(a,b); } 227c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 228c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f por<Packet4f>(const Packet4f& a, const Packet4f& b) 229c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 230c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return vreinterpretq_f32_u32(vorrq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b))); 231c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 232c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i por<Packet4i>(const Packet4i& a, const Packet4i& b) { return vorrq_s32(a,b); } 233c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 234c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b) 235c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 236c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b))); 237c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 238c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(const Packet4i& a, const Packet4i& b) { return veorq_s32(a,b); } 239c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 240c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b) 241c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 242c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return vreinterpretq_f32_u32(vbicq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b))); 243c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 244c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) { return vbicq_s32(a,b); } 245c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 2462b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_f32(from); } 2472b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int32_t* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_s32(from); } 248c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 2492b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_f32(from); } 2502b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int32_t* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_s32(from); } 251c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 2522b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from) 253c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 254c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath float32x2_t lo, hi; 2557faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez lo = vld1_dup_f32(from); 2567faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez hi = vld1_dup_f32(from+1); 257c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return vcombine_f32(lo, hi); 258c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 2592b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int32_t* from) 260c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 261c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath int32x2_t lo, hi; 2627faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez lo = vld1_dup_s32(from); 2637faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez hi = vld1_dup_s32(from+1); 264c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return vcombine_s32(lo, hi); 265c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 266c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 2672b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE void pstore<float> (float* to, const Packet4f& from) { EIGEN_DEBUG_ALIGNED_STORE vst1q_f32(to, from); } 2682b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE void pstore<int32_t>(int32_t* to, const Packet4i& from) { EIGEN_DEBUG_ALIGNED_STORE vst1q_s32(to, from); } 2692b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 2702b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE void pstoreu<float> (float* to, const Packet4f& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_f32(to, from); } 2712b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE void pstoreu<int32_t>(int32_t* to, const Packet4i& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_s32(to, from); } 272c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 2732b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_DEVICE_FUNC inline Packet4f pgather<float, Packet4f>(const float* from, Index stride) 2742b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 2752b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Packet4f res = pset1<Packet4f>(0.f); 2762b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang res = vsetq_lane_f32(from[0*stride], res, 0); 2772b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang res = vsetq_lane_f32(from[1*stride], res, 1); 2782b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang res = vsetq_lane_f32(from[2*stride], res, 2); 2792b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang res = vsetq_lane_f32(from[3*stride], res, 3); 2802b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return res; 2812b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 2822b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_DEVICE_FUNC inline Packet4i pgather<int32_t, Packet4i>(const int32_t* from, Index stride) 2832b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 2842b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Packet4i res = pset1<Packet4i>(0); 2852b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang res = vsetq_lane_s32(from[0*stride], res, 0); 2862b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang res = vsetq_lane_s32(from[1*stride], res, 1); 2872b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang res = vsetq_lane_s32(from[2*stride], res, 2); 2882b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang res = vsetq_lane_s32(from[3*stride], res, 3); 2892b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return res; 2902b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 2912b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 2922b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_DEVICE_FUNC inline void pscatter<float, Packet4f>(float* to, const Packet4f& from, Index stride) 2932b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 2942b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang to[stride*0] = vgetq_lane_f32(from, 0); 2952b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang to[stride*1] = vgetq_lane_f32(from, 1); 2962b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang to[stride*2] = vgetq_lane_f32(from, 2); 2972b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang to[stride*3] = vgetq_lane_f32(from, 3); 2982b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 2992b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_DEVICE_FUNC inline void pscatter<int32_t, Packet4i>(int32_t* to, const Packet4i& from, Index stride) 3002b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 3012b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang to[stride*0] = vgetq_lane_s32(from, 0); 3022b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang to[stride*1] = vgetq_lane_s32(from, 1); 3032b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang to[stride*2] = vgetq_lane_s32(from, 2); 3042b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang to[stride*3] = vgetq_lane_s32(from, 3); 3052b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 306c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 3072b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE void prefetch<float> (const float* addr) { EIGEN_ARM_PREFETCH(addr); } 3082b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE void prefetch<int32_t>(const int32_t* addr) { EIGEN_ARM_PREFETCH(addr); } 309c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 310c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// FIXME only store the 2 first elements ? 3112b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { float EIGEN_ALIGN16 x[4]; vst1q_f32(x, a); return x[0]; } 3122b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE int32_t pfirst<Packet4i>(const Packet4i& a) { int32_t EIGEN_ALIGN16 x[4]; vst1q_s32(x, a); return x[0]; } 313c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 314c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) { 315c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath float32x2_t a_lo, a_hi; 316c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath Packet4f a_r64; 317c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 318c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath a_r64 = vrev64q_f32(a); 319c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath a_lo = vget_low_f32(a_r64); 320c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath a_hi = vget_high_f32(a_r64); 321c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return vcombine_f32(a_hi, a_lo); 322c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 323c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a) { 324c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath int32x2_t a_lo, a_hi; 325c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath Packet4i a_r64; 326c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 327c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath a_r64 = vrev64q_s32(a); 328c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath a_lo = vget_low_s32(a_r64); 329c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath a_hi = vget_high_s32(a_r64); 330c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return vcombine_s32(a_hi, a_lo); 331c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 3322b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 333c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a) { return vabsq_f32(a); } 334c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) { return vabsq_s32(a); } 335c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 336c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a) 337c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 338c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath float32x2_t a_lo, a_hi, sum; 339c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 340c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath a_lo = vget_low_f32(a); 341c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath a_hi = vget_high_f32(a); 342c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath sum = vpadd_f32(a_lo, a_hi); 343c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath sum = vpadd_f32(sum, sum); 3447faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez return vget_lane_f32(sum, 0); 345c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 346c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 347c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(const Packet4f* vecs) 348c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 349c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath float32x4x2_t vtrn1, vtrn2, res1, res2; 350c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath Packet4f sum1, sum2, sum; 351c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 352c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // NEON zip performs interleaving of the supplied vectors. 353c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // We perform two interleaves in a row to acquire the transposed vector 354c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath vtrn1 = vzipq_f32(vecs[0], vecs[2]); 355c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath vtrn2 = vzipq_f32(vecs[1], vecs[3]); 356c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath res1 = vzipq_f32(vtrn1.val[0], vtrn2.val[0]); 357c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath res2 = vzipq_f32(vtrn1.val[1], vtrn2.val[1]); 358c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 359c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // Do the addition of the resulting vectors 360c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath sum1 = vaddq_f32(res1.val[0], res1.val[1]); 361c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath sum2 = vaddq_f32(res2.val[0], res2.val[1]); 362c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath sum = vaddq_f32(sum1, sum2); 363c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 364c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return sum; 365c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 366c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 3672b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE int32_t predux<Packet4i>(const Packet4i& a) 368c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 369c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath int32x2_t a_lo, a_hi, sum; 370c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 371c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath a_lo = vget_low_s32(a); 372c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath a_hi = vget_high_s32(a); 373c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath sum = vpadd_s32(a_lo, a_hi); 374c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath sum = vpadd_s32(sum, sum); 3757faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez return vget_lane_s32(sum, 0); 376c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 377c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 378c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE Packet4i preduxp<Packet4i>(const Packet4i* vecs) 379c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 380c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath int32x4x2_t vtrn1, vtrn2, res1, res2; 381c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath Packet4i sum1, sum2, sum; 382c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 383c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // NEON zip performs interleaving of the supplied vectors. 384c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // We perform two interleaves in a row to acquire the transposed vector 385c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath vtrn1 = vzipq_s32(vecs[0], vecs[2]); 386c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath vtrn2 = vzipq_s32(vecs[1], vecs[3]); 387c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath res1 = vzipq_s32(vtrn1.val[0], vtrn2.val[0]); 388c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath res2 = vzipq_s32(vtrn1.val[1], vtrn2.val[1]); 389c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 390c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // Do the addition of the resulting vectors 391c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath sum1 = vaddq_s32(res1.val[0], res1.val[1]); 392c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath sum2 = vaddq_s32(res2.val[0], res2.val[1]); 393c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath sum = vaddq_s32(sum1, sum2); 394c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 395c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath return sum; 396c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 397c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 398c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// Other reduction functions: 399c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// mul 400c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a) 401c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 402c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath float32x2_t a_lo, a_hi, prod; 403c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 404c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // Get a_lo = |a1|a2| and a_hi = |a3|a4| 405c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath a_lo = vget_low_f32(a); 406c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath a_hi = vget_high_f32(a); 407c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // Get the product of a_lo * a_hi -> |a1*a3|a2*a4| 408c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath prod = vmul_f32(a_lo, a_hi); 409c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // Multiply prod with its swapped value |a2*a4|a1*a3| 410c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath prod = vmul_f32(prod, vrev64_f32(prod)); 411c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 4127faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez return vget_lane_f32(prod, 0); 413c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 4142b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE int32_t predux_mul<Packet4i>(const Packet4i& a) 415c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 416c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath int32x2_t a_lo, a_hi, prod; 417c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 418c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // Get a_lo = |a1|a2| and a_hi = |a3|a4| 419c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath a_lo = vget_low_s32(a); 420c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath a_hi = vget_high_s32(a); 421c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // Get the product of a_lo * a_hi -> |a1*a3|a2*a4| 422c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath prod = vmul_s32(a_lo, a_hi); 423c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath // Multiply prod with its swapped value |a2*a4|a1*a3| 424c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath prod = vmul_s32(prod, vrev64_s32(prod)); 425c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 4267faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez return vget_lane_s32(prod, 0); 427c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 428c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 429c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// min 430c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a) 431c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 432c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath float32x2_t a_lo, a_hi, min; 433c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 434c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath a_lo = vget_low_f32(a); 435c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath a_hi = vget_high_f32(a); 436c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath min = vpmin_f32(a_lo, a_hi); 437c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath min = vpmin_f32(min, min); 438c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 4397faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez return vget_lane_f32(min, 0); 440c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 4417faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez 4422b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE int32_t predux_min<Packet4i>(const Packet4i& a) 443c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 444c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath int32x2_t a_lo, a_hi, min; 445c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 446c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath a_lo = vget_low_s32(a); 447c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath a_hi = vget_high_s32(a); 448c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath min = vpmin_s32(a_lo, a_hi); 449c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath min = vpmin_s32(min, min); 4507faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez 4517faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez return vget_lane_s32(min, 0); 452c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 453c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 454c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// max 455c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<> EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a) 456c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 457c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath float32x2_t a_lo, a_hi, max; 458c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 459c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath a_lo = vget_low_f32(a); 460c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath a_hi = vget_high_f32(a); 461c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath max = vpmax_f32(a_lo, a_hi); 462c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath max = vpmax_f32(max, max); 463c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 4647faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez return vget_lane_f32(max, 0); 465c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 4667faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez 4672b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE int32_t predux_max<Packet4i>(const Packet4i& a) 468c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{ 469c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath int32x2_t a_lo, a_hi, max; 470c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 471c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath a_lo = vget_low_s32(a); 472c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath a_hi = vget_high_s32(a); 473c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath max = vpmax_s32(a_lo, a_hi); 4742b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang max = vpmax_s32(max, max); 475c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 4767faaa9f3f0df9d23790277834d426c3d992ac3baCarlos Hernandez return vget_lane_s32(max, 0); 477c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} 478c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 479c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// this PALIGN_NEON business is to work around a bug in LLVM Clang 3.0 causing incorrect compilation errors, 480c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath// see bug 347 and this LLVM bug: http://llvm.org/bugs/show_bug.cgi?id=11074 481c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#define PALIGN_NEON(Offset,Type,Command) \ 482c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathtemplate<>\ 483c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamathstruct palign_impl<Offset,Type>\ 484c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath{\ 485c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath EIGEN_STRONG_INLINE static void run(Type& first, const Type& second)\ 486c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath {\ 487c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath if (Offset!=0)\ 488c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath first = Command(first, second, Offset);\ 489c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath }\ 490c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath};\ 491c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 492c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan KamathPALIGN_NEON(0,Packet4f,vextq_f32) 493c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan KamathPALIGN_NEON(1,Packet4f,vextq_f32) 494c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan KamathPALIGN_NEON(2,Packet4f,vextq_f32) 495c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan KamathPALIGN_NEON(3,Packet4f,vextq_f32) 496c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan KamathPALIGN_NEON(0,Packet4i,vextq_s32) 497c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan KamathPALIGN_NEON(1,Packet4i,vextq_s32) 498c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan KamathPALIGN_NEON(2,Packet4i,vextq_s32) 499c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan KamathPALIGN_NEON(3,Packet4i,vextq_s32) 5002b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 501c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#undef PALIGN_NEON 502c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 5032b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao WangEIGEN_DEVICE_FUNC inline void 5042b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangptranspose(PacketBlock<Packet4f,4>& kernel) { 5052b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang float32x4x2_t tmp1 = vzipq_f32(kernel.packet[0], kernel.packet[1]); 5062b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang float32x4x2_t tmp2 = vzipq_f32(kernel.packet[2], kernel.packet[3]); 5072b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 5082b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang kernel.packet[0] = vcombine_f32(vget_low_f32(tmp1.val[0]), vget_low_f32(tmp2.val[0])); 5092b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang kernel.packet[1] = vcombine_f32(vget_high_f32(tmp1.val[0]), vget_high_f32(tmp2.val[0])); 5102b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang kernel.packet[2] = vcombine_f32(vget_low_f32(tmp1.val[1]), vget_low_f32(tmp2.val[1])); 5112b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang kernel.packet[3] = vcombine_f32(vget_high_f32(tmp1.val[1]), vget_high_f32(tmp2.val[1])); 5122b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 5132b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 5142b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao WangEIGEN_DEVICE_FUNC inline void 5152b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangptranspose(PacketBlock<Packet4i,4>& kernel) { 5162b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang int32x4x2_t tmp1 = vzipq_s32(kernel.packet[0], kernel.packet[1]); 5172b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang int32x4x2_t tmp2 = vzipq_s32(kernel.packet[2], kernel.packet[3]); 5182b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang kernel.packet[0] = vcombine_s32(vget_low_s32(tmp1.val[0]), vget_low_s32(tmp2.val[0])); 5192b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang kernel.packet[1] = vcombine_s32(vget_high_s32(tmp1.val[0]), vget_high_s32(tmp2.val[0])); 5202b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang kernel.packet[2] = vcombine_s32(vget_low_s32(tmp1.val[1]), vget_low_s32(tmp2.val[1])); 5212b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang kernel.packet[3] = vcombine_s32(vget_high_s32(tmp1.val[1]), vget_high_s32(tmp2.val[1])); 5222b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 5232b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 5242b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang//---------- double ---------- 5252b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 5262b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// Clang 3.5 in the iOS toolchain has an ICE triggered by NEON intrisics for double. 5272b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// Confirmed at least with __apple_build_version__ = 6000054. 5282b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifdef __apple_build_version__ 5292b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// Let's hope that by the time __apple_build_version__ hits the 601* range, the bug will be fixed. 5302b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// https://gist.github.com/yamaya/2924292 suggests that the 3 first digits are only updated with 5312b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// major toolchain updates. 5322b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#define EIGEN_APPLE_DOUBLE_NEON_BUG (__apple_build_version__ < 6010000) 5332b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#else 5342b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#define EIGEN_APPLE_DOUBLE_NEON_BUG 0 5352b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 5362b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 5372b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#if EIGEN_ARCH_ARM64 && !EIGEN_APPLE_DOUBLE_NEON_BUG 5382b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 5392b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// Bug 907: workaround missing declarations of the following two functions in the ADK 5402b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// Defining these functions as templates ensures that if these intrinsics are 5412b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// already defined in arm_neon.h, then our workaround doesn't cause a conflict 5422b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// and has lower priority in overload resolution. 5432b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate <typename T> 5442b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wanguint64x2_t vreinterpretq_u64_f64(T a) 5452b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 5462b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return (uint64x2_t) a; 5472b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 5482b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 5492b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate <typename T> 5502b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangfloat64x2_t vreinterpretq_f64_u64(T a) 5512b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 5522b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return (float64x2_t) a; 5532b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 5542b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 5552b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtypedef float64x2_t Packet2d; 5562b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtypedef float64x1_t Packet1d; 5572b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 5582b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> struct packet_traits<double> : default_packet_traits 5592b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 5602b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang typedef Packet2d type; 5612b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang typedef Packet2d half; 5622b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang enum { 5632b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Vectorizable = 1, 5642b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang AlignedOnScalar = 1, 5652b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang size = 2, 5662b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasHalfPacket=0, 5672b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 5682b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasDiv = 1, 5692b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang // FIXME check the Has* 5702b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasSin = 0, 5712b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasCos = 0, 5722b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasLog = 0, 5732b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasExp = 0, 5742b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang HasSqrt = 0 5752b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang }; 5762b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang}; 5772b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 5782b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> struct unpacket_traits<Packet2d> { typedef double type; enum {size=2, alignment=Aligned16}; typedef Packet2d half; }; 5792b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 5802b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) { return vdupq_n_f64(from); } 5812b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 5822b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(const double& a) 5832b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 5842b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang const double countdown_raw[] = {0.0,1.0}; 5852b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang const Packet2d countdown = vld1q_f64(countdown_raw); 5862b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return vaddq_f64(pset1<Packet2d>(a), countdown); 5872b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 5882b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(const Packet2d& a, const Packet2d& b) { return vaddq_f64(a,b); } 5892b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 5902b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(const Packet2d& a, const Packet2d& b) { return vsubq_f64(a,b); } 5912b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 5922b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a) { return vnegq_f64(a); } 5932b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 5942b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d pconj(const Packet2d& a) { return a; } 5952b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 5962b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(const Packet2d& a, const Packet2d& b) { return vmulq_f64(a,b); } 5972b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 5982b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(const Packet2d& a, const Packet2d& b) { return vdivq_f64(a,b); } 5992b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 6002b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#ifdef __ARM_FEATURE_FMA 6012b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// See bug 936. See above comment about FMA for float. 6022b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) { return vfmaq_f64(c,a,b); } 6032b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#else 6042b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) { return vmlaq_f64(c,a,b); } 6052b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 6062b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 6072b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b) { return vminq_f64(a,b); } 6082b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 6092b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(const Packet2d& a, const Packet2d& b) { return vmaxq_f64(a,b); } 6102b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 6112b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// Logical Operations are not supported for float, so we have to reinterpret casts using NEON intrinsics 6122b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(const Packet2d& a, const Packet2d& b) 6132b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 6142b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return vreinterpretq_f64_u64(vandq_u64(vreinterpretq_u64_f64(a),vreinterpretq_u64_f64(b))); 6152b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 6162b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 6172b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d por<Packet2d>(const Packet2d& a, const Packet2d& b) 6182b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 6192b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return vreinterpretq_f64_u64(vorrq_u64(vreinterpretq_u64_f64(a),vreinterpretq_u64_f64(b))); 6202b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 6212b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 6222b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& a, const Packet2d& b) 6232b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 6242b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return vreinterpretq_f64_u64(veorq_u64(vreinterpretq_u64_f64(a),vreinterpretq_u64_f64(b))); 6252b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 6262b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 6272b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(const Packet2d& a, const Packet2d& b) 6282b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 6292b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return vreinterpretq_f64_u64(vbicq_u64(vreinterpretq_u64_f64(a),vreinterpretq_u64_f64(b))); 6302b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 6312b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 6322b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_f64(from); } 6332b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 6342b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_f64(from); } 6352b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 6362b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(const double* from) 6372b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 6382b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return vld1q_dup_f64(from); 6392b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 6402b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet2d& from) { EIGEN_DEBUG_ALIGNED_STORE vst1q_f64(to, from); } 6412b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 6422b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet2d& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_f64(to, from); } 6432b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 6442b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_DEVICE_FUNC inline Packet2d pgather<double, Packet2d>(const double* from, Index stride) 6452b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 6462b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang Packet2d res = pset1<Packet2d>(0.0); 6472b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang res = vsetq_lane_f64(from[0*stride], res, 0); 6482b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang res = vsetq_lane_f64(from[1*stride], res, 1); 6492b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return res; 6502b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 6512b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_DEVICE_FUNC inline void pscatter<double, Packet2d>(double* to, const Packet2d& from, Index stride) 6522b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 6532b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang to[stride*0] = vgetq_lane_f64(from, 0); 6542b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang to[stride*1] = vgetq_lane_f64(from, 1); 6552b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 6562b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { EIGEN_ARM_PREFETCH(addr); } 6572b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 6582b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// FIXME only store the 2 first elements ? 6592b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { return vgetq_lane_f64(a, 0); } 6602b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 6612b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a) { return vcombine_f64(vget_high_f64(a), vget_low_f64(a)); } 6622b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 6632b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) { return vabsq_f64(a); } 6642b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 6652b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#if EIGEN_COMP_CLANG && defined(__apple_build_version__) 6662b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// workaround ICE, see bug 907 6672b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a) { return (vget_low_f64(a) + vget_high_f64(a))[0]; } 6682b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#else 6692b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a) { return vget_lane_f64(vget_low_f64(a) + vget_high_f64(a), 0); } 6702b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 6712b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 6722b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE Packet2d preduxp<Packet2d>(const Packet2d* vecs) 6732b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{ 6742b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang float64x2_t trn1, trn2; 6752b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 6762b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang // NEON zip performs interleaving of the supplied vectors. 6772b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang // We perform two interleaves in a row to acquire the transposed vector 6782b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang trn1 = vzip1q_f64(vecs[0], vecs[1]); 6792b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang trn2 = vzip2q_f64(vecs[0], vecs[1]); 6802b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 6812b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang // Do the addition of the resulting vectors 6822b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang return vaddq_f64(trn1, trn2); 6832b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 6842b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// Other reduction functions: 6852b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// mul 6862b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#if EIGEN_COMP_CLANG && defined(__apple_build_version__) 6872b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a) { return (vget_low_f64(a) * vget_high_f64(a))[0]; } 6882b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#else 6892b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a) { return vget_lane_f64(vget_low_f64(a) * vget_high_f64(a), 0); } 6902b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif 6912b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 6922b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// min 6932b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE double predux_min<Packet2d>(const Packet2d& a) { return vgetq_lane_f64(vpminq_f64(a, a), 0); } 6942b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 6952b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// max 6962b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<> EIGEN_STRONG_INLINE double predux_max<Packet2d>(const Packet2d& a) { return vgetq_lane_f64(vpmaxq_f64(a, a), 0); } 6972b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 6982b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// this PALIGN_NEON business is to work around a bug in LLVM Clang 3.0 causing incorrect compilation errors, 6992b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang// see bug 347 and this LLVM bug: http://llvm.org/bugs/show_bug.cgi?id=11074 7002b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#define PALIGN_NEON(Offset,Type,Command) \ 7012b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangtemplate<>\ 7022b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangstruct palign_impl<Offset,Type>\ 7032b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang{\ 7042b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang EIGEN_STRONG_INLINE static void run(Type& first, const Type& second)\ 7052b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang {\ 7062b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang if (Offset!=0)\ 7072b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang first = Command(first, second, Offset);\ 7082b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang }\ 7092b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang};\ 7102b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 7112b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao WangPALIGN_NEON(0,Packet2d,vextq_f64) 7122b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao WangPALIGN_NEON(1,Packet2d,vextq_f64) 7132b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#undef PALIGN_NEON 7142b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 7152b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao WangEIGEN_DEVICE_FUNC inline void 7162b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wangptranspose(PacketBlock<Packet2d,2>& kernel) { 7172b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang float64x2_t trn1 = vzip1q_f64(kernel.packet[0], kernel.packet[1]); 7182b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang float64x2_t trn2 = vzip2q_f64(kernel.packet[0], kernel.packet[1]); 7192b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 7202b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang kernel.packet[0] = trn1; 7212b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang kernel.packet[1] = trn2; 7222b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang} 7232b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang#endif // EIGEN_ARCH_ARM64 7242b8756b6f1de65d3f8bffab45be6c44ceb7411fcMiao Wang 725c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} // end namespace internal 726c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 727c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath} // end namespace Eigen 728c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath 729c981c48f5bc9aefeffc0bcb0cc3934c2fae179ddNarayan Kamath#endif // EIGEN_PACKET_MATH_NEON_H 730