CGBuiltin.cpp revision 6bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89
1022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson//===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===// 2022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson// 3022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson// The LLVM Compiler Infrastructure 4022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson// 50bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// This file is distributed under the University of Illinois Open Source 60bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// License. See LICENSE.TXT for details. 7022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson// 8022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson//===----------------------------------------------------------------------===// 9022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson// 10022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson// This contains code to emit Builtin calls as LLVM code. 11022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson// 12022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson//===----------------------------------------------------------------------===// 13022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson 14022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson#include "CodeGenFunction.h" 1555bcace250e1ff366e4482714b344b8cbc8be5f3Fariborz Jahanian#include "CGObjCRuntime.h" 1655fc873017f10f6f566b182b70f6fc22aefa3464Chandler Carruth#include "CodeGenModule.h" 1755fc873017f10f6f566b182b70f6fc22aefa3464Chandler Carruth#include "TargetInfo.h" 18bef20ac367a09555b30d6eb3847a81ec164caf88Chris Lattner#include "clang/AST/ASTContext.h" 19c4a1dea2dc56bd1357ec91b829a0b9e68229a13eDaniel Dunbar#include "clang/AST/Decl.h" 206b15cdc1312f8fc45c86ee75e2a85106700e97f6Chris Lattner#include "clang/Basic/TargetBuiltins.h" 2155fc873017f10f6f566b182b70f6fc22aefa3464Chandler Carruth#include "clang/Basic/TargetInfo.h" 228b54999a831bb195c08541ca995ef0505c96193fMark Lacey#include "clang/CodeGen/CGFunctionInfo.h" 233b844ba7d5be205a9b4f5f0b0d1b7978977f4b8cChandler Carruth#include "llvm/IR/DataLayout.h" 243b844ba7d5be205a9b4f5f0b0d1b7978977f4b8cChandler Carruth#include "llvm/IR/Intrinsics.h" 25558229f267d2cdee69639df04382082e7c7ad54cJakub Staszak 26022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlssonusing namespace clang; 27022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlssonusing namespace CodeGen; 28ca6fcfad547dcec3fdd17790b4fab0918df74b37Anders Carlssonusing namespace llvm; 29ca6fcfad547dcec3fdd17790b4fab0918df74b37Anders Carlsson 30a45680b7e7c49ea9893c6cff585984f3e4120366John McCall/// getBuiltinLibFunction - Given a builtin id for a function like 31a45680b7e7c49ea9893c6cff585984f3e4120366John McCall/// "__builtin_fabsf", return a Function* for "fabsf". 32a45680b7e7c49ea9893c6cff585984f3e4120366John McCallllvm::Value *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD, 33a45680b7e7c49ea9893c6cff585984f3e4120366John McCall unsigned BuiltinID) { 34a45680b7e7c49ea9893c6cff585984f3e4120366John McCall assert(Context.BuiltinInfo.isLibFunction(BuiltinID)); 35a45680b7e7c49ea9893c6cff585984f3e4120366John McCall 36a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // Get the name, skip over the __builtin_ prefix (if necessary). 37a45680b7e7c49ea9893c6cff585984f3e4120366John McCall StringRef Name; 38a45680b7e7c49ea9893c6cff585984f3e4120366John McCall GlobalDecl D(FD); 39a45680b7e7c49ea9893c6cff585984f3e4120366John McCall 40a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // If the builtin has been declared explicitly with an assembler label, 41a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // use the mangled name. This differs from the plain label on platforms 42a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // that prefix labels. 43a45680b7e7c49ea9893c6cff585984f3e4120366John McCall if (FD->hasAttr<AsmLabelAttr>()) 44a45680b7e7c49ea9893c6cff585984f3e4120366John McCall Name = getMangledName(D); 45a45680b7e7c49ea9893c6cff585984f3e4120366John McCall else 46a45680b7e7c49ea9893c6cff585984f3e4120366John McCall Name = Context.BuiltinInfo.GetName(BuiltinID) + 10; 47a45680b7e7c49ea9893c6cff585984f3e4120366John McCall 48a45680b7e7c49ea9893c6cff585984f3e4120366John McCall llvm::FunctionType *Ty = 49a45680b7e7c49ea9893c6cff585984f3e4120366John McCall cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType())); 50a45680b7e7c49ea9893c6cff585984f3e4120366John McCall 51a45680b7e7c49ea9893c6cff585984f3e4120366John McCall return GetOrCreateLLVMFunction(Name, Ty, D, /*ForVTable=*/false); 52a45680b7e7c49ea9893c6cff585984f3e4120366John McCall} 53a45680b7e7c49ea9893c6cff585984f3e4120366John McCall 5426815d97c5743481e317f17a8d53a6819d061862John McCall/// Emit the conversions required to turn the given value into an 5526815d97c5743481e317f17a8d53a6819d061862John McCall/// integer of the given size. 5626815d97c5743481e317f17a8d53a6819d061862John McCallstatic Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V, 572acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner QualType T, llvm::IntegerType *IntType) { 5826815d97c5743481e317f17a8d53a6819d061862John McCall V = CGF.EmitToMemory(V, T); 5926815d97c5743481e317f17a8d53a6819d061862John McCall 6026815d97c5743481e317f17a8d53a6819d061862John McCall if (V->getType()->isPointerTy()) 6126815d97c5743481e317f17a8d53a6819d061862John McCall return CGF.Builder.CreatePtrToInt(V, IntType); 6226815d97c5743481e317f17a8d53a6819d061862John McCall 6326815d97c5743481e317f17a8d53a6819d061862John McCall assert(V->getType() == IntType); 6426815d97c5743481e317f17a8d53a6819d061862John McCall return V; 65db4325b098eff5e9e660db19f0148423fb21f27fChandler Carruth} 66db4325b098eff5e9e660db19f0148423fb21f27fChandler Carruth 6726815d97c5743481e317f17a8d53a6819d061862John McCallstatic Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V, 682acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner QualType T, llvm::Type *ResultType) { 6926815d97c5743481e317f17a8d53a6819d061862John McCall V = CGF.EmitFromMemory(V, T); 7026815d97c5743481e317f17a8d53a6819d061862John McCall 7126815d97c5743481e317f17a8d53a6819d061862John McCall if (ResultType->isPointerTy()) 7226815d97c5743481e317f17a8d53a6819d061862John McCall return CGF.Builder.CreateIntToPtr(V, ResultType); 7326815d97c5743481e317f17a8d53a6819d061862John McCall 7426815d97c5743481e317f17a8d53a6819d061862John McCall assert(V->getType() == ResultType); 7526815d97c5743481e317f17a8d53a6819d061862John McCall return V; 76db4325b098eff5e9e660db19f0148423fb21f27fChandler Carruth} 77db4325b098eff5e9e660db19f0148423fb21f27fChandler Carruth 780002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar/// Utility to insert an atomic instruction based on Instrinsic::ID 790002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar/// and the expression node. 80cb61a7bbe635cfa941b1aeaaa1fbda1bf900ee51Daniel Dunbarstatic RValue EmitBinaryAtomic(CodeGenFunction &CGF, 81c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman llvm::AtomicRMWInst::BinOp Kind, 82c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman const CallExpr *E) { 8326815d97c5743481e317f17a8d53a6819d061862John McCall QualType T = E->getType(); 8426815d97c5743481e317f17a8d53a6819d061862John McCall assert(E->getArg(0)->getType()->isPointerType()); 8526815d97c5743481e317f17a8d53a6819d061862John McCall assert(CGF.getContext().hasSameUnqualifiedType(T, 8626815d97c5743481e317f17a8d53a6819d061862John McCall E->getArg(0)->getType()->getPointeeType())); 8726815d97c5743481e317f17a8d53a6819d061862John McCall assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType())); 8826815d97c5743481e317f17a8d53a6819d061862John McCall 894f209445c06a43283c6f72dda7c925538b1578e9Chris Lattner llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0)); 90956a5a17713deb1b5b27893303c4f308a1bd2a62Micah Villmow unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); 9126815d97c5743481e317f17a8d53a6819d061862John McCall 929cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::IntegerType *IntType = 93db4325b098eff5e9e660db19f0148423fb21f27fChandler Carruth llvm::IntegerType::get(CGF.getLLVMContext(), 9426815d97c5743481e317f17a8d53a6819d061862John McCall CGF.getContext().getTypeSize(T)); 959cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); 9626815d97c5743481e317f17a8d53a6819d061862John McCall 9726815d97c5743481e317f17a8d53a6819d061862John McCall llvm::Value *Args[2]; 9826815d97c5743481e317f17a8d53a6819d061862John McCall Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType); 9926815d97c5743481e317f17a8d53a6819d061862John McCall Args[1] = CGF.EmitScalarExpr(E->getArg(1)); 1002acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ValueType = Args[1]->getType(); 10126815d97c5743481e317f17a8d53a6819d061862John McCall Args[1] = EmitToInt(CGF, Args[1], T, IntType); 10226815d97c5743481e317f17a8d53a6819d061862John McCall 103c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman llvm::Value *Result = 104c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman CGF.Builder.CreateAtomicRMW(Kind, Args[0], Args[1], 105c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman llvm::SequentiallyConsistent); 10626815d97c5743481e317f17a8d53a6819d061862John McCall Result = EmitFromInt(CGF, Result, T, ValueType); 10726815d97c5743481e317f17a8d53a6819d061862John McCall return RValue::get(Result); 1080002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar} 1090002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar 1100002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar/// Utility to insert an atomic instruction based Instrinsic::ID and 11126815d97c5743481e317f17a8d53a6819d061862John McCall/// the expression node, where the return value is the result of the 11226815d97c5743481e317f17a8d53a6819d061862John McCall/// operation. 113420b11850d3f4557421f43f519b59d528329c668Chris Lattnerstatic RValue EmitBinaryAtomicPost(CodeGenFunction &CGF, 114c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman llvm::AtomicRMWInst::BinOp Kind, 115c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman const CallExpr *E, 1160002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar Instruction::BinaryOps Op) { 11726815d97c5743481e317f17a8d53a6819d061862John McCall QualType T = E->getType(); 11826815d97c5743481e317f17a8d53a6819d061862John McCall assert(E->getArg(0)->getType()->isPointerType()); 11926815d97c5743481e317f17a8d53a6819d061862John McCall assert(CGF.getContext().hasSameUnqualifiedType(T, 12026815d97c5743481e317f17a8d53a6819d061862John McCall E->getArg(0)->getType()->getPointeeType())); 12126815d97c5743481e317f17a8d53a6819d061862John McCall assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType())); 12226815d97c5743481e317f17a8d53a6819d061862John McCall 1234f209445c06a43283c6f72dda7c925538b1578e9Chris Lattner llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0)); 124956a5a17713deb1b5b27893303c4f308a1bd2a62Micah Villmow unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); 12526815d97c5743481e317f17a8d53a6819d061862John McCall 1269cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::IntegerType *IntType = 127db4325b098eff5e9e660db19f0148423fb21f27fChandler Carruth llvm::IntegerType::get(CGF.getLLVMContext(), 12826815d97c5743481e317f17a8d53a6819d061862John McCall CGF.getContext().getTypeSize(T)); 1299cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); 13026815d97c5743481e317f17a8d53a6819d061862John McCall 13126815d97c5743481e317f17a8d53a6819d061862John McCall llvm::Value *Args[2]; 13226815d97c5743481e317f17a8d53a6819d061862John McCall Args[1] = CGF.EmitScalarExpr(E->getArg(1)); 1332acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ValueType = Args[1]->getType(); 13426815d97c5743481e317f17a8d53a6819d061862John McCall Args[1] = EmitToInt(CGF, Args[1], T, IntType); 13526815d97c5743481e317f17a8d53a6819d061862John McCall Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType); 13626815d97c5743481e317f17a8d53a6819d061862John McCall 137c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman llvm::Value *Result = 138c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman CGF.Builder.CreateAtomicRMW(Kind, Args[0], Args[1], 139c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman llvm::SequentiallyConsistent); 14026815d97c5743481e317f17a8d53a6819d061862John McCall Result = CGF.Builder.CreateBinOp(Op, Result, Args[1]); 14126815d97c5743481e317f17a8d53a6819d061862John McCall Result = EmitFromInt(CGF, Result, T, ValueType); 14226815d97c5743481e317f17a8d53a6819d061862John McCall return RValue::get(Result); 1431ffe281890f3cd7728316b45a1f3dd4d3120af7bMon P Wang} 1441ffe281890f3cd7728316b45a1f3dd4d3120af7bMon P Wang 145420b11850d3f4557421f43f519b59d528329c668Chris Lattner/// EmitFAbs - Emit a call to fabs/fabsf/fabsl, depending on the type of ValTy, 146420b11850d3f4557421f43f519b59d528329c668Chris Lattner/// which must be a scalar floating point type. 147420b11850d3f4557421f43f519b59d528329c668Chris Lattnerstatic Value *EmitFAbs(CodeGenFunction &CGF, Value *V, QualType ValTy) { 148420b11850d3f4557421f43f519b59d528329c668Chris Lattner const BuiltinType *ValTyP = ValTy->getAs<BuiltinType>(); 149420b11850d3f4557421f43f519b59d528329c668Chris Lattner assert(ValTyP && "isn't scalar fp type!"); 150258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 151420b11850d3f4557421f43f519b59d528329c668Chris Lattner StringRef FnName; 152420b11850d3f4557421f43f519b59d528329c668Chris Lattner switch (ValTyP->getKind()) { 153b219cfc4d75f0a03630b7c4509ef791b7e97b2c8David Blaikie default: llvm_unreachable("Isn't a scalar fp type!"); 154420b11850d3f4557421f43f519b59d528329c668Chris Lattner case BuiltinType::Float: FnName = "fabsf"; break; 155420b11850d3f4557421f43f519b59d528329c668Chris Lattner case BuiltinType::Double: FnName = "fabs"; break; 156420b11850d3f4557421f43f519b59d528329c668Chris Lattner case BuiltinType::LongDouble: FnName = "fabsl"; break; 157420b11850d3f4557421f43f519b59d528329c668Chris Lattner } 158258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 159420b11850d3f4557421f43f519b59d528329c668Chris Lattner // The prototype is something that takes and returns whatever V's type is. 160da549e8995c447542d5631b8b67fcc3a9582797aJay Foad llvm::FunctionType *FT = llvm::FunctionType::get(V->getType(), V->getType(), 16195d318c4c10437db40ca6e15fdf32e04012da58eBenjamin Kramer false); 162420b11850d3f4557421f43f519b59d528329c668Chris Lattner llvm::Value *Fn = CGF.CGM.CreateRuntimeFunction(FT, FnName); 163420b11850d3f4557421f43f519b59d528329c668Chris Lattner 164b71757b2e02b83c18b736729a747016f0b1e1353Benjamin Kramer return CGF.EmitNounwindRuntimeCall(Fn, V, "abs"); 165420b11850d3f4557421f43f519b59d528329c668Chris Lattner} 166420b11850d3f4557421f43f519b59d528329c668Chris Lattner 167a45680b7e7c49ea9893c6cff585984f3e4120366John McCallstatic RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *Fn, 168a45680b7e7c49ea9893c6cff585984f3e4120366John McCall const CallExpr *E, llvm::Value *calleeValue) { 169b914e87377fd4d7642f544000a79f8648c6f06c9Peter Collingbourne return CGF.EmitCall(E->getCallee()->getType(), calleeValue, E->getLocStart(), 170a45680b7e7c49ea9893c6cff585984f3e4120366John McCall ReturnValueSlot(), E->arg_begin(), E->arg_end(), Fn); 171a45680b7e7c49ea9893c6cff585984f3e4120366John McCall} 172a45680b7e7c49ea9893c6cff585984f3e4120366John McCall 1730cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// \brief Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.* 1740cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// depending on IntrinsicID. 1750cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// 1760cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// \arg CGF The current codegen function. 1770cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// \arg IntrinsicID The ID for the Intrinsic we wish to generate. 1780cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// \arg X The first argument to the llvm.*.with.overflow.*. 1790cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// \arg Y The second argument to the llvm.*.with.overflow.*. 1800cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// \arg Carry The carry returned by the llvm.*.with.overflow.*. 1810cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// \returns The result (i.e. sum/product) returned by the intrinsic. 1820cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesmanstatic llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF, 1830cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman const llvm::Intrinsic::ID IntrinsicID, 1840cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *X, llvm::Value *Y, 1850cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *&Carry) { 1860cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // Make sure we have integers of the same width. 1870cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman assert(X->getType() == Y->getType() && 1880cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman "Arguments must be the same type. (Did you forget to make sure both " 1890cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman "arguments have the same integer width?)"); 1900cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman 191563fb903fab15972e2d9c66b0ae046a94be86a71NAKAMURA Takumi llvm::Value *Callee = CGF.CGM.getIntrinsic(IntrinsicID, X->getType()); 1920cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *Tmp = CGF.Builder.CreateCall2(Callee, X, Y); 1930cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman Carry = CGF.Builder.CreateExtractValue(Tmp, 1); 1940cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman return CGF.Builder.CreateExtractValue(Tmp, 0); 1950cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman} 1960cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman 1971eb4433ac451dc16f4133a88af2d002ac26c58efMike StumpRValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, 198ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar unsigned BuiltinID, const CallExpr *E) { 199564ea2a99b3afeac9ded332730a56db1f6358a58Chris Lattner // See if we can constant fold this builtin. If so, don't emit it at all. 200f35d35a2316dcb65d078844696c2032b71a7f103Anders Carlsson Expr::EvalResult Result; 20152a27f5be98d99fd5339949d8a3d0b2aec655e0bEli Friedman if (E->EvaluateAsRValue(Result, CGM.getContext()) && 202dd697bc8629f0fa6777245610d52ca7ceb3b67f4Fariborz Jahanian !Result.hasSideEffects()) { 203f35d35a2316dcb65d078844696c2032b71a7f103Anders Carlsson if (Result.Val.isInt()) 204d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall return RValue::get(llvm::ConstantInt::get(getLLVMContext(), 2054a28d5deeba33722aa009eab488591fb9055cc7eOwen Anderson Result.Val.getInt())); 206a1aa9e36e6e21f74c56cf9e72cb5bd9aa2a92fd4Chris Lattner if (Result.Val.isFloat()) 207d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall return RValue::get(llvm::ConstantFP::get(getLLVMContext(), 208d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Result.Val.getFloat())); 2091f32999ec79a980576e100d64d5f3267eb19ea49Chris Lattner } 2101eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 211564ea2a99b3afeac9ded332730a56db1f6358a58Chris Lattner switch (BuiltinID) { 212564ea2a99b3afeac9ded332730a56db1f6358a58Chris Lattner default: break; // Handle intrinsics and libm functions below. 213506ff88f44562df267b6a06608ab841b76df2a2bChris Lattner case Builtin::BI__builtin___CFStringMakeConstantString: 2140d13f6fdbdd6f06e2449b8834dda53334abd399aDavid Chisnall case Builtin::BI__builtin___NSStringMakeConstantString: 2156bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(CGM.EmitConstantExpr(E, E->getType(), nullptr)); 2166a705f0443f8398ece14d80e71d8e8c9e71aa84aChris Lattner case Builtin::BI__builtin_stdarg_start: 217793680ed8104bf088d1b382b963a8badcb3f07deAnders Carlsson case Builtin::BI__builtin_va_start: 218651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case Builtin::BI__va_start: 219793680ed8104bf088d1b382b963a8badcb3f07deAnders Carlsson case Builtin::BI__builtin_va_end: { 220651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *ArgValue = (BuiltinID == Builtin::BI__va_start) 221651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ? EmitScalarExpr(E->getArg(0)) 222651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : EmitVAListRef(E->getArg(0)); 2232acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *DestType = Int8PtrTy; 224793680ed8104bf088d1b382b963a8badcb3f07deAnders Carlsson if (ArgValue->getType() != DestType) 2251eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump ArgValue = Builder.CreateBitCast(ArgValue, DestType, 226b27ffbef8e8aa1e87b63cbc0d9cd630aba30dae5Daniel Dunbar ArgValue->getName().data()); 227793680ed8104bf088d1b382b963a8badcb3f07deAnders Carlsson 2281eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump Intrinsic::ID inst = (BuiltinID == Builtin::BI__builtin_va_end) ? 2296a705f0443f8398ece14d80e71d8e8c9e71aa84aChris Lattner Intrinsic::vaend : Intrinsic::vastart; 2307acda7c4a0e4aec6c003b3169ca45a5f3bc7e033Chris Lattner return RValue::get(Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue)); 231793680ed8104bf088d1b382b963a8badcb3f07deAnders Carlsson } 232a28ef8b5c6ea452472d89041128fac8b683fe968Anders Carlsson case Builtin::BI__builtin_va_copy: { 2334fd0aa5803357d8c72eeac2cae15e12649ea08feEli Friedman Value *DstPtr = EmitVAListRef(E->getArg(0)); 2344fd0aa5803357d8c72eeac2cae15e12649ea08feEli Friedman Value *SrcPtr = EmitVAListRef(E->getArg(1)); 235a28ef8b5c6ea452472d89041128fac8b683fe968Anders Carlsson 2362acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *Type = Int8PtrTy; 237a28ef8b5c6ea452472d89041128fac8b683fe968Anders Carlsson 238a28ef8b5c6ea452472d89041128fac8b683fe968Anders Carlsson DstPtr = Builder.CreateBitCast(DstPtr, Type); 239a28ef8b5c6ea452472d89041128fac8b683fe968Anders Carlsson SrcPtr = Builder.CreateBitCast(SrcPtr, Type); 2401eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump return RValue::get(Builder.CreateCall2(CGM.getIntrinsic(Intrinsic::vacopy), 2413eae03e7d165f20a863a9a4d7122ba2a691ab16dChris Lattner DstPtr, SrcPtr)); 242a28ef8b5c6ea452472d89041128fac8b683fe968Anders Carlsson } 243258f930227c1a102c9c22eee88df65f748863425Jim Grosbach case Builtin::BI__builtin_abs: 244f7b2d8b3b1971e73e2d2f0662520dc7693235a7fEli Friedman case Builtin::BI__builtin_labs: 245f7b2d8b3b1971e73e2d2f0662520dc7693235a7fEli Friedman case Builtin::BI__builtin_llabs: { 2461eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump Value *ArgValue = EmitScalarExpr(E->getArg(0)); 2471eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2489a847f598c4907a72b8593b364b9e6b94b086e75Chris Lattner Value *NegOp = Builder.CreateNeg(ArgValue, "neg"); 2491eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump Value *CmpResult = 2501eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump Builder.CreateICmpSGE(ArgValue, 251c9c88b4159791c48e486ca94e3743b5979e2b7a6Owen Anderson llvm::Constant::getNullValue(ArgValue->getType()), 2529a847f598c4907a72b8593b364b9e6b94b086e75Chris Lattner "abscond"); 2531eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump Value *Result = 254c2251dc59b0edc28f9303637dec970a7520939adAnders Carlsson Builder.CreateSelect(CmpResult, ArgValue, NegOp, "abs"); 2551eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 256c2251dc59b0edc28f9303637dec970a7520939adAnders Carlsson return RValue::get(Result); 257c2251dc59b0edc28f9303637dec970a7520939adAnders Carlsson } 258258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 259ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian case Builtin::BI__builtin_conj: 260ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian case Builtin::BI__builtin_conjf: 261ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian case Builtin::BI__builtin_conjl: { 262ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0)); 263ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian Value *Real = ComplexVal.first; 264ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian Value *Imag = ComplexVal.second; 265258f930227c1a102c9c22eee88df65f748863425Jim Grosbach Value *Zero = 266258f930227c1a102c9c22eee88df65f748863425Jim Grosbach Imag->getType()->isFPOrFPVectorTy() 267ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian ? llvm::ConstantFP::getZeroValueForNegation(Imag->getType()) 268ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian : llvm::Constant::getNullValue(Imag->getType()); 269258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 270ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian Imag = Builder.CreateFSub(Zero, Imag, "sub"); 271ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian return RValue::getComplex(std::make_pair(Real, Imag)); 272ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian } 273ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian case Builtin::BI__builtin_creal: 274ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian case Builtin::BI__builtin_crealf: 27508cc03fcd97c3886b004d861aadbc28de70092c5Meador Inge case Builtin::BI__builtin_creall: 27608cc03fcd97c3886b004d861aadbc28de70092c5Meador Inge case Builtin::BIcreal: 27708cc03fcd97c3886b004d861aadbc28de70092c5Meador Inge case Builtin::BIcrealf: 27808cc03fcd97c3886b004d861aadbc28de70092c5Meador Inge case Builtin::BIcreall: { 279ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0)); 280ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian return RValue::get(ComplexVal.first); 281ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian } 282258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 283ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian case Builtin::BI__builtin_cimag: 284ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian case Builtin::BI__builtin_cimagf: 28508cc03fcd97c3886b004d861aadbc28de70092c5Meador Inge case Builtin::BI__builtin_cimagl: 28608cc03fcd97c3886b004d861aadbc28de70092c5Meador Inge case Builtin::BIcimag: 28708cc03fcd97c3886b004d861aadbc28de70092c5Meador Inge case Builtin::BIcimagf: 28808cc03fcd97c3886b004d861aadbc28de70092c5Meador Inge case Builtin::BIcimagl: { 289ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0)); 290ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian return RValue::get(ComplexVal.second); 291ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian } 292258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 293a49a2839266f0bac2b6286e9f49fdc0c292938feBenjamin Kramer case Builtin::BI__builtin_ctzs: 2943a31d60cffedfb7c9e6d129a5c9ba15fa74f179aAnders Carlsson case Builtin::BI__builtin_ctz: 2953a31d60cffedfb7c9e6d129a5c9ba15fa74f179aAnders Carlsson case Builtin::BI__builtin_ctzl: 2963a31d60cffedfb7c9e6d129a5c9ba15fa74f179aAnders Carlsson case Builtin::BI__builtin_ctzll: { 2973a31d60cffedfb7c9e6d129a5c9ba15fa74f179aAnders Carlsson Value *ArgValue = EmitScalarExpr(E->getArg(0)); 2981eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2999cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *ArgType = ArgValue->getType(); 3008dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType); 3013a31d60cffedfb7c9e6d129a5c9ba15fa74f179aAnders Carlsson 3022acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ResultType = ConvertType(E->getType()); 30364aa4b3ec7e62288e2e66c1935487ece995ca94bJohn McCall Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef()); 3048b30a9379f730875ba8fb2d0fe2d43611e0c20ffBob Wilson Value *Result = Builder.CreateCall2(F, ArgValue, ZeroUndef); 3053a31d60cffedfb7c9e6d129a5c9ba15fa74f179aAnders Carlsson if (Result->getType() != ResultType) 306eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 307eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands "cast"); 3083a31d60cffedfb7c9e6d129a5c9ba15fa74f179aAnders Carlsson return RValue::get(Result); 3093a31d60cffedfb7c9e6d129a5c9ba15fa74f179aAnders Carlsson } 310a49a2839266f0bac2b6286e9f49fdc0c292938feBenjamin Kramer case Builtin::BI__builtin_clzs: 311f4e853340590d5c32e58379e8c379ea1777d3101Eli Friedman case Builtin::BI__builtin_clz: 312f4e853340590d5c32e58379e8c379ea1777d3101Eli Friedman case Builtin::BI__builtin_clzl: 313f4e853340590d5c32e58379e8c379ea1777d3101Eli Friedman case Builtin::BI__builtin_clzll: { 314f4e853340590d5c32e58379e8c379ea1777d3101Eli Friedman Value *ArgValue = EmitScalarExpr(E->getArg(0)); 3151eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3169cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *ArgType = ArgValue->getType(); 3178dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType); 318f4e853340590d5c32e58379e8c379ea1777d3101Eli Friedman 3192acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ResultType = ConvertType(E->getType()); 32064aa4b3ec7e62288e2e66c1935487ece995ca94bJohn McCall Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef()); 3218b30a9379f730875ba8fb2d0fe2d43611e0c20ffBob Wilson Value *Result = Builder.CreateCall2(F, ArgValue, ZeroUndef); 322f4e853340590d5c32e58379e8c379ea1777d3101Eli Friedman if (Result->getType() != ResultType) 323eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 324eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands "cast"); 325f4e853340590d5c32e58379e8c379ea1777d3101Eli Friedman return RValue::get(Result); 326f4e853340590d5c32e58379e8c379ea1777d3101Eli Friedman } 32704b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_ffs: 32804b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_ffsl: 32904b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_ffsll: { 33004b290030eee33295600728450f348989d1a627eDaniel Dunbar // ffs(x) -> x ? cttz(x) + 1 : 0 33104b290030eee33295600728450f348989d1a627eDaniel Dunbar Value *ArgValue = EmitScalarExpr(E->getArg(0)); 3321eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3339cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *ArgType = ArgValue->getType(); 3348dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType); 3351eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3362acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ResultType = ConvertType(E->getType()); 33750058ec63f38eabeb94391a61d2e7fb18a062173Chandler Carruth Value *Tmp = Builder.CreateAdd(Builder.CreateCall2(F, ArgValue, 33850058ec63f38eabeb94391a61d2e7fb18a062173Chandler Carruth Builder.getTrue()), 339578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer llvm::ConstantInt::get(ArgType, 1)); 340c9c88b4159791c48e486ca94e3743b5979e2b7a6Owen Anderson Value *Zero = llvm::Constant::getNullValue(ArgType); 34104b290030eee33295600728450f348989d1a627eDaniel Dunbar Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero"); 34204b290030eee33295600728450f348989d1a627eDaniel Dunbar Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp, "ffs"); 34304b290030eee33295600728450f348989d1a627eDaniel Dunbar if (Result->getType() != ResultType) 344eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 345eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands "cast"); 34604b290030eee33295600728450f348989d1a627eDaniel Dunbar return RValue::get(Result); 34704b290030eee33295600728450f348989d1a627eDaniel Dunbar } 34804b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_parity: 34904b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_parityl: 35004b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_parityll: { 35104b290030eee33295600728450f348989d1a627eDaniel Dunbar // parity(x) -> ctpop(x) & 1 35204b290030eee33295600728450f348989d1a627eDaniel Dunbar Value *ArgValue = EmitScalarExpr(E->getArg(0)); 3531eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3549cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *ArgType = ArgValue->getType(); 3558dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType); 3561eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3572acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ResultType = ConvertType(E->getType()); 358578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer Value *Tmp = Builder.CreateCall(F, ArgValue); 359578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1)); 36004b290030eee33295600728450f348989d1a627eDaniel Dunbar if (Result->getType() != ResultType) 361eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 362eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands "cast"); 36304b290030eee33295600728450f348989d1a627eDaniel Dunbar return RValue::get(Result); 36404b290030eee33295600728450f348989d1a627eDaniel Dunbar } 36504b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_popcount: 36604b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_popcountl: 36704b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_popcountll: { 36804b290030eee33295600728450f348989d1a627eDaniel Dunbar Value *ArgValue = EmitScalarExpr(E->getArg(0)); 3691eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3709cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *ArgType = ArgValue->getType(); 3718dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType); 3721eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3732acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ResultType = ConvertType(E->getType()); 374578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer Value *Result = Builder.CreateCall(F, ArgValue); 37504b290030eee33295600728450f348989d1a627eDaniel Dunbar if (Result->getType() != ResultType) 376eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 377eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands "cast"); 37804b290030eee33295600728450f348989d1a627eDaniel Dunbar return RValue::get(Result); 37904b290030eee33295600728450f348989d1a627eDaniel Dunbar } 380e42b8a596886fc98e367c73e54d761446700029eFariborz Jahanian case Builtin::BI__builtin_expect: { 381dd697bc8629f0fa6777245610d52ca7ceb3b67f4Fariborz Jahanian Value *ArgValue = EmitScalarExpr(E->getArg(0)); 3829cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *ArgType = ArgValue->getType(); 383558229f267d2cdee69639df04382082e7c7ad54cJakub Staszak 3848dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType); 385558229f267d2cdee69639df04382082e7c7ad54cJakub Staszak Value *ExpectedValue = EmitScalarExpr(E->getArg(1)); 386558229f267d2cdee69639df04382082e7c7ad54cJakub Staszak 387558229f267d2cdee69639df04382082e7c7ad54cJakub Staszak Value *Result = Builder.CreateCall2(FnExpect, ArgValue, ExpectedValue, 388558229f267d2cdee69639df04382082e7c7ad54cJakub Staszak "expval"); 389558229f267d2cdee69639df04382082e7c7ad54cJakub Staszak return RValue::get(Result); 390e42b8a596886fc98e367c73e54d761446700029eFariborz Jahanian } 391d190057934331390ff67ebf51d66186dd5e392f0Benjamin Kramer case Builtin::BI__builtin_bswap16: 392df4852ac816e6050d53b808b86d7c1c9738eb99eAnders Carlsson case Builtin::BI__builtin_bswap32: 393df4852ac816e6050d53b808b86d7c1c9738eb99eAnders Carlsson case Builtin::BI__builtin_bswap64: { 3941feedd84221e8dbcc3faf3de27cc42b559db845dChris Lattner Value *ArgValue = EmitScalarExpr(E->getArg(0)); 3959cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *ArgType = ArgValue->getType(); 3968dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::bswap, ArgType); 397578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer return RValue::get(Builder.CreateCall(F, ArgValue)); 3981eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } 399d5f8a4fd4d6dfb0415b93bb7ab721bba5cab1332Daniel Dunbar case Builtin::BI__builtin_object_size: { 400c6794850a570a91c5f224b6f0293db9f560f4213Richard Smith // We rely on constant folding to deal with expressions with side effects. 401c6794850a570a91c5f224b6f0293db9f560f4213Richard Smith assert(!E->getArg(0)->HasSideEffects(getContext()) && 402c6794850a570a91c5f224b6f0293db9f560f4213Richard Smith "should have been constant folded"); 403c6794850a570a91c5f224b6f0293db9f560f4213Richard Smith 404b16d32f74ffc467a5604934a1f844906be20cf7dMike Stump // We pass this builtin onto the optimizer so that it can 405b16d32f74ffc467a5604934a1f844906be20cf7dMike Stump // figure out the object size in more complex cases. 4068dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer llvm::Type *ResType = ConvertType(E->getType()); 407258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 408fee667f35e64751baa7fefe70b4e7bab06c8cd86Eric Christopher // LLVM only supports 0 and 2, make sure that we pass along that 409fee667f35e64751baa7fefe70b4e7bab06c8cd86Eric Christopher // as a boolean. 410fee667f35e64751baa7fefe70b4e7bab06c8cd86Eric Christopher Value *Ty = EmitScalarExpr(E->getArg(1)); 411fee667f35e64751baa7fefe70b4e7bab06c8cd86Eric Christopher ConstantInt *CI = dyn_cast<ConstantInt>(Ty); 412fee667f35e64751baa7fefe70b4e7bab06c8cd86Eric Christopher assert(CI); 413fee667f35e64751baa7fefe70b4e7bab06c8cd86Eric Christopher uint64_t val = CI->getZExtValue(); 414258f930227c1a102c9c22eee88df65f748863425Jim Grosbach CI = ConstantInt::get(Builder.getInt1Ty(), (val & 0x2) >> 1); 4158dd4baacf584b1fc7b0e2c79a0b8b197a4fb5fabMatt Arsenault // FIXME: Get right address space. 4168dd4baacf584b1fc7b0e2c79a0b8b197a4fb5fabMatt Arsenault llvm::Type *Tys[] = { ResType, Builder.getInt8PtrTy(0) }; 4178dd4baacf584b1fc7b0e2c79a0b8b197a4fb5fabMatt Arsenault Value *F = CGM.getIntrinsic(Intrinsic::objectsize, Tys); 4183e86a0433db4c664d29f2b19eb977138e071a68aNuno Lopes return RValue::get(Builder.CreateCall2(F, EmitScalarExpr(E->getArg(0)),CI)); 419d5f8a4fd4d6dfb0415b93bb7ab721bba5cab1332Daniel Dunbar } 4204493f79fce48cd9cbd9f55fa9d452cde736747a0Daniel Dunbar case Builtin::BI__builtin_prefetch: { 4214493f79fce48cd9cbd9f55fa9d452cde736747a0Daniel Dunbar Value *Locality, *RW, *Address = EmitScalarExpr(E->getArg(0)); 4224493f79fce48cd9cbd9f55fa9d452cde736747a0Daniel Dunbar // FIXME: Technically these constants should of type 'int', yes? 4231eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump RW = (E->getNumArgs() > 1) ? EmitScalarExpr(E->getArg(1)) : 42477b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner llvm::ConstantInt::get(Int32Ty, 0); 4251eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump Locality = (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) : 42677b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner llvm::ConstantInt::get(Int32Ty, 3); 4272eccb672799d19fb535ce349566fea6729cbb891Bruno Cardoso Lopes Value *Data = llvm::ConstantInt::get(Int32Ty, 1); 4288dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::prefetch); 4292eccb672799d19fb535ce349566fea6729cbb891Bruno Cardoso Lopes return RValue::get(Builder.CreateCall4(F, Address, RW, Locality, Data)); 4304493f79fce48cd9cbd9f55fa9d452cde736747a0Daniel Dunbar } 431a841c19f7860393d6319bf40e9d662284462771dHal Finkel case Builtin::BI__builtin_readcyclecounter: { 432a841c19f7860393d6319bf40e9d662284462771dHal Finkel Value *F = CGM.getIntrinsic(Intrinsic::readcyclecounter); 433a841c19f7860393d6319bf40e9d662284462771dHal Finkel return RValue::get(Builder.CreateCall(F)); 434a841c19f7860393d6319bf40e9d662284462771dHal Finkel } 435651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case Builtin::BI__builtin___clear_cache: { 436651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Begin = EmitScalarExpr(E->getArg(0)); 437651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *End = EmitScalarExpr(E->getArg(1)); 438651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *F = CGM.getIntrinsic(Intrinsic::clear_cache); 439651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return RValue::get(Builder.CreateCall2(F, Begin, End)); 440651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4414493f79fce48cd9cbd9f55fa9d452cde736747a0Daniel Dunbar case Builtin::BI__builtin_trap: { 4428dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::trap); 4434493f79fce48cd9cbd9f55fa9d452cde736747a0Daniel Dunbar return RValue::get(Builder.CreateCall(F)); 444df4852ac816e6050d53b808b86d7c1c9738eb99eAnders Carlsson } 445385cc5f01b59af0183a825351d93695ca7185091Nico Weber case Builtin::BI__debugbreak: { 446385cc5f01b59af0183a825351d93695ca7185091Nico Weber Value *F = CGM.getIntrinsic(Intrinsic::debugtrap); 447385cc5f01b59af0183a825351d93695ca7185091Nico Weber return RValue::get(Builder.CreateCall(F)); 448385cc5f01b59af0183a825351d93695ca7185091Nico Weber } 44921190d54634d6e244e85d28ad915ce2fe86ecbffChris Lattner case Builtin::BI__builtin_unreachable: { 4504f45bc099f2665bc6e4bcbb169aa452390dbf3feWill Dietz if (SanOpts->Unreachable) 4514def70d3040e73707c738f7c366737a986135edfRichard Smith EmitCheck(Builder.getFalse(), "builtin_unreachable", 4524def70d3040e73707c738f7c366737a986135edfRichard Smith EmitCheckSourceLocation(E->getExprLoc()), 453cfa88f893915ceb8ae4ce2f17c46c24a4d67502fDmitri Gribenko ArrayRef<llvm::Value *>(), CRK_Unrecoverable); 454cd5b22e12b6513163dd131589746c194090f14e6John McCall else 455cd5b22e12b6513163dd131589746c194090f14e6John McCall Builder.CreateUnreachable(); 456cd5b22e12b6513163dd131589746c194090f14e6John McCall 457cd5b22e12b6513163dd131589746c194090f14e6John McCall // We do need to preserve an insertion point. 458d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall EmitBlock(createBasicBlock("unreachable.cont")); 459cd5b22e12b6513163dd131589746c194090f14e6John McCall 4606bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 46121190d54634d6e244e85d28ad915ce2fe86ecbffChris Lattner } 462258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 463a933c3c052bbd87b01cc6fc7a7745e1c4b1757fbDaniel Dunbar case Builtin::BI__builtin_powi: 464a933c3c052bbd87b01cc6fc7a7745e1c4b1757fbDaniel Dunbar case Builtin::BI__builtin_powif: 465a933c3c052bbd87b01cc6fc7a7745e1c4b1757fbDaniel Dunbar case Builtin::BI__builtin_powil: { 466a933c3c052bbd87b01cc6fc7a7745e1c4b1757fbDaniel Dunbar Value *Base = EmitScalarExpr(E->getArg(0)); 467a933c3c052bbd87b01cc6fc7a7745e1c4b1757fbDaniel Dunbar Value *Exponent = EmitScalarExpr(E->getArg(1)); 4689cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *ArgType = Base->getType(); 4698dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::powi, ArgType); 470578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer return RValue::get(Builder.CreateCall2(F, Base, Exponent)); 471a933c3c052bbd87b01cc6fc7a7745e1c4b1757fbDaniel Dunbar } 472a933c3c052bbd87b01cc6fc7a7745e1c4b1757fbDaniel Dunbar 473fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_isgreater: 474fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_isgreaterequal: 475fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_isless: 476fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_islessequal: 477fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_islessgreater: 478fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_isunordered: { 479fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner // Ordered comparisons: we know the arguments to these are matching scalar 480fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner // floating point values. 4811eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump Value *LHS = EmitScalarExpr(E->getArg(0)); 482fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner Value *RHS = EmitScalarExpr(E->getArg(1)); 4831eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 484fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner switch (BuiltinID) { 485b219cfc4d75f0a03630b7c4509ef791b7e97b2c8David Blaikie default: llvm_unreachable("Unknown ordered comparison"); 486fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_isgreater: 487fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner LHS = Builder.CreateFCmpOGT(LHS, RHS, "cmp"); 488fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner break; 489fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_isgreaterequal: 490fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner LHS = Builder.CreateFCmpOGE(LHS, RHS, "cmp"); 491fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner break; 492fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_isless: 493fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner LHS = Builder.CreateFCmpOLT(LHS, RHS, "cmp"); 494fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner break; 495fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_islessequal: 496fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner LHS = Builder.CreateFCmpOLE(LHS, RHS, "cmp"); 497fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner break; 498fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_islessgreater: 499fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner LHS = Builder.CreateFCmpONE(LHS, RHS, "cmp"); 500fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner break; 5011eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump case Builtin::BI__builtin_isunordered: 502fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner LHS = Builder.CreateFCmpUNO(LHS, RHS, "cmp"); 503fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner break; 504fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner } 505fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner // ZExt bool to int type. 506578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer return RValue::get(Builder.CreateZExt(LHS, ConvertType(E->getType()))); 507fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner } 508d6139895f43d161a972d134ffda4229d2f548eb6Eli Friedman case Builtin::BI__builtin_isnan: { 509d6139895f43d161a972d134ffda4229d2f548eb6Eli Friedman Value *V = EmitScalarExpr(E->getArg(0)); 510d6139895f43d161a972d134ffda4229d2f548eb6Eli Friedman V = Builder.CreateFCmpUNO(V, V, "cmp"); 511578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); 512d6139895f43d161a972d134ffda4229d2f548eb6Eli Friedman } 513258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 514420b11850d3f4557421f43f519b59d528329c668Chris Lattner case Builtin::BI__builtin_isinf: { 515420b11850d3f4557421f43f519b59d528329c668Chris Lattner // isinf(x) --> fabs(x) == infinity 516420b11850d3f4557421f43f519b59d528329c668Chris Lattner Value *V = EmitScalarExpr(E->getArg(0)); 517420b11850d3f4557421f43f519b59d528329c668Chris Lattner V = EmitFAbs(*this, V, E->getArg(0)->getType()); 518258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 519420b11850d3f4557421f43f519b59d528329c668Chris Lattner V = Builder.CreateFCmpOEQ(V, ConstantFP::getInfinity(V->getType()),"isinf"); 520578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); 521420b11850d3f4557421f43f519b59d528329c668Chris Lattner } 522258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 52358ae5b4b1e2fc02b95d6650af5755a59639aa153Chris Lattner // TODO: BI__builtin_isinf_sign 52458ae5b4b1e2fc02b95d6650af5755a59639aa153Chris Lattner // isinf_sign(x) -> isinf(x) ? (signbit(x) ? -1 : 1) : 0 5256349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer 5266349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer case Builtin::BI__builtin_isnormal: { 5276349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer // isnormal(x) --> x == x && fabsf(x) < infinity && fabsf(x) >= float_min 5286349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer Value *V = EmitScalarExpr(E->getArg(0)); 5296349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq"); 5306349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer 5316349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer Value *Abs = EmitFAbs(*this, V, E->getArg(0)->getType()); 5326349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer Value *IsLessThanInf = 5336349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer Builder.CreateFCmpULT(Abs, ConstantFP::getInfinity(V->getType()),"isinf"); 5346349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer APFloat Smallest = APFloat::getSmallestNormalized( 5356349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer getContext().getFloatTypeSemantics(E->getArg(0)->getType())); 5366349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer Value *IsNormal = 5376349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer Builder.CreateFCmpUGE(Abs, ConstantFP::get(V->getContext(), Smallest), 5386349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer "isnormal"); 5396349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer V = Builder.CreateAnd(Eq, IsLessThanInf, "and"); 5406349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer V = Builder.CreateAnd(V, IsNormal, "and"); 5416349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); 5426349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer } 5436349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer 544ed074150c9a775c5e2e1c4ececeba18ba880ce7dChris Lattner case Builtin::BI__builtin_isfinite: { 545ef004ec5adb0e11815cef3435fa5ac7366d783a9Julien Lerouge // isfinite(x) --> x == x && fabs(x) != infinity; 546ed074150c9a775c5e2e1c4ececeba18ba880ce7dChris Lattner Value *V = EmitScalarExpr(E->getArg(0)); 547ed074150c9a775c5e2e1c4ececeba18ba880ce7dChris Lattner Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq"); 548258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 549ed074150c9a775c5e2e1c4ececeba18ba880ce7dChris Lattner Value *Abs = EmitFAbs(*this, V, E->getArg(0)->getType()); 550ed074150c9a775c5e2e1c4ececeba18ba880ce7dChris Lattner Value *IsNotInf = 551ed074150c9a775c5e2e1c4ececeba18ba880ce7dChris Lattner Builder.CreateFCmpUNE(Abs, ConstantFP::getInfinity(V->getType()),"isinf"); 552258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 553ed074150c9a775c5e2e1c4ececeba18ba880ce7dChris Lattner V = Builder.CreateAnd(Eq, IsNotInf, "and"); 554ed074150c9a775c5e2e1c4ececeba18ba880ce7dChris Lattner return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); 555ed074150c9a775c5e2e1c4ececeba18ba880ce7dChris Lattner } 5567867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer 5577867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer case Builtin::BI__builtin_fpclassify: { 5587867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *V = EmitScalarExpr(E->getArg(5)); 5592acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *Ty = ConvertType(E->getArg(5)->getType()); 5607867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer 5617867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer // Create Result 5627867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer BasicBlock *Begin = Builder.GetInsertBlock(); 5637867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer BasicBlock *End = createBasicBlock("fpclassify_end", this->CurFn); 5647867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.SetInsertPoint(End); 5657867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer PHINode *Result = 566bbf3bacb3e0c1ebb3e8a4a8b1330404a7e379315Jay Foad Builder.CreatePHI(ConvertType(E->getArg(0)->getType()), 4, 5677867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer "fpclassify_result"); 5687867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer 5697867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer // if (V==0) return FP_ZERO 5707867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.SetInsertPoint(Begin); 5717867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *IsZero = Builder.CreateFCmpOEQ(V, Constant::getNullValue(Ty), 5727867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer "iszero"); 5737867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *ZeroLiteral = EmitScalarExpr(E->getArg(4)); 5747867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer BasicBlock *NotZero = createBasicBlock("fpclassify_not_zero", this->CurFn); 5757867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.CreateCondBr(IsZero, End, NotZero); 5767867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Result->addIncoming(ZeroLiteral, Begin); 5777867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer 5787867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer // if (V != V) return FP_NAN 5797867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.SetInsertPoint(NotZero); 5807867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *IsNan = Builder.CreateFCmpUNO(V, V, "cmp"); 5817867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *NanLiteral = EmitScalarExpr(E->getArg(0)); 5827867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer BasicBlock *NotNan = createBasicBlock("fpclassify_not_nan", this->CurFn); 5837867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.CreateCondBr(IsNan, End, NotNan); 5847867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Result->addIncoming(NanLiteral, NotZero); 5857867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer 5867867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer // if (fabs(V) == infinity) return FP_INFINITY 5877867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.SetInsertPoint(NotNan); 5887867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *VAbs = EmitFAbs(*this, V, E->getArg(5)->getType()); 5897867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *IsInf = 5907867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(V->getType()), 5917867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer "isinf"); 5927867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *InfLiteral = EmitScalarExpr(E->getArg(1)); 5937867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer BasicBlock *NotInf = createBasicBlock("fpclassify_not_inf", this->CurFn); 5947867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.CreateCondBr(IsInf, End, NotInf); 5957867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Result->addIncoming(InfLiteral, NotNan); 5967867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer 5977867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer // if (fabs(V) >= MIN_NORMAL) return FP_NORMAL else FP_SUBNORMAL 5987867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.SetInsertPoint(NotInf); 5997867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer APFloat Smallest = APFloat::getSmallestNormalized( 6007867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer getContext().getFloatTypeSemantics(E->getArg(5)->getType())); 6017867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *IsNormal = 6027867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.CreateFCmpUGE(VAbs, ConstantFP::get(V->getContext(), Smallest), 6037867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer "isnormal"); 6047867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *NormalResult = 6057867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.CreateSelect(IsNormal, EmitScalarExpr(E->getArg(2)), 6067867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer EmitScalarExpr(E->getArg(3))); 6077867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.CreateBr(End); 6087867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Result->addIncoming(NormalResult, NotInf); 6097867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer 6107867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer // return Result 6117867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.SetInsertPoint(End); 6127867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer return RValue::get(Result); 6137867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer } 614258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 615b52fe9ce99970955a5f581f5c66fcd89be9a268bEli Friedman case Builtin::BIalloca: 6166526de425dbfd368407566ff123b5bdc0924859aReid Kleckner case Builtin::BI_alloca: 6179e800e3dd80d77f6c47054738177bf824089f55aChris Lattner case Builtin::BI__builtin_alloca: { 6189e800e3dd80d77f6c47054738177bf824089f55aChris Lattner Value *Size = EmitScalarExpr(E->getArg(0)); 619578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer return RValue::get(Builder.CreateAlloca(Builder.getInt8Ty(), Size)); 6201caae959017b355e9bb61250d5a0d04edbf468b0Daniel Dunbar } 621e6dddfd907f6ea58daed5e26eeaacd893d98db9bEli Friedman case Builtin::BIbzero: 6221caae959017b355e9bb61250d5a0d04edbf468b0Daniel Dunbar case Builtin::BI__builtin_bzero: { 623ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman std::pair<llvm::Value*, unsigned> Dest = 624ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman EmitPointerWithAlignment(E->getArg(0)); 6253ecd785aff34381f3704d9cb28fe3ef85af759deMon P Wang Value *SizeVal = EmitScalarExpr(E->getArg(1)); 626ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman Builder.CreateMemSet(Dest.first, Builder.getInt8(0), SizeVal, 627ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman Dest.second, false); 628ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman return RValue::get(Dest.first); 6299e800e3dd80d77f6c47054738177bf824089f55aChris Lattner } 630e6ec205d6d0f4aec27bf49ca1e8fbb139acc2f2bEli Friedman case Builtin::BImemcpy: 631d4b32e46517358f34e8cfbea35010adfcc3786e0Eli Friedman case Builtin::BI__builtin_memcpy: { 632ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman std::pair<llvm::Value*, unsigned> Dest = 633ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman EmitPointerWithAlignment(E->getArg(0)); 634ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman std::pair<llvm::Value*, unsigned> Src = 635ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman EmitPointerWithAlignment(E->getArg(1)); 6363ecd785aff34381f3704d9cb28fe3ef85af759deMon P Wang Value *SizeVal = EmitScalarExpr(E->getArg(2)); 637ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman unsigned Align = std::min(Dest.second, Src.second); 638ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman Builder.CreateMemCpy(Dest.first, Src.first, SizeVal, Align, false); 639ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman return RValue::get(Dest.first); 6401caae959017b355e9bb61250d5a0d04edbf468b0Daniel Dunbar } 641258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 642a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner case Builtin::BI__builtin___memcpy_chk: { 643f3477c13eeaf11b32a41f181398fb5deffd0dd73Sylvestre Ledru // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memcpy iff cst1<=cst2. 644a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith llvm::APSInt Size, DstSize; 645a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) || 646a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext())) 647a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner break; 648a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner if (Size.ugt(DstSize)) 649a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner break; 650ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman std::pair<llvm::Value*, unsigned> Dest = 651ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman EmitPointerWithAlignment(E->getArg(0)); 652ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman std::pair<llvm::Value*, unsigned> Src = 653ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman EmitPointerWithAlignment(E->getArg(1)); 654a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); 655ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman unsigned Align = std::min(Dest.second, Src.second); 656ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman Builder.CreateMemCpy(Dest.first, Src.first, SizeVal, Align, false); 657ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman return RValue::get(Dest.first); 658a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner } 659258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 6608e2eab27056a78bf1db50ee09929438ed5ea9d93Fariborz Jahanian case Builtin::BI__builtin_objc_memmove_collectable: { 66155bcace250e1ff366e4482714b344b8cbc8be5f3Fariborz Jahanian Value *Address = EmitScalarExpr(E->getArg(0)); 66255bcace250e1ff366e4482714b344b8cbc8be5f3Fariborz Jahanian Value *SrcAddr = EmitScalarExpr(E->getArg(1)); 66355bcace250e1ff366e4482714b344b8cbc8be5f3Fariborz Jahanian Value *SizeVal = EmitScalarExpr(E->getArg(2)); 664258f930227c1a102c9c22eee88df65f748863425Jim Grosbach CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, 66555bcace250e1ff366e4482714b344b8cbc8be5f3Fariborz Jahanian Address, SrcAddr, SizeVal); 66655bcace250e1ff366e4482714b344b8cbc8be5f3Fariborz Jahanian return RValue::get(Address); 66755bcace250e1ff366e4482714b344b8cbc8be5f3Fariborz Jahanian } 668a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner 669a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner case Builtin::BI__builtin___memmove_chk: { 670f3477c13eeaf11b32a41f181398fb5deffd0dd73Sylvestre Ledru // fold __builtin_memmove_chk(x, y, cst1, cst2) to memmove iff cst1<=cst2. 671a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith llvm::APSInt Size, DstSize; 672a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) || 673a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext())) 674a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner break; 675a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner if (Size.ugt(DstSize)) 676a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner break; 677ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman std::pair<llvm::Value*, unsigned> Dest = 678ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman EmitPointerWithAlignment(E->getArg(0)); 679ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman std::pair<llvm::Value*, unsigned> Src = 680ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman EmitPointerWithAlignment(E->getArg(1)); 681a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); 682ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman unsigned Align = std::min(Dest.second, Src.second); 683ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman Builder.CreateMemMove(Dest.first, Src.first, SizeVal, Align, false); 684ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman return RValue::get(Dest.first); 685a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner } 686a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner 687e6ec205d6d0f4aec27bf49ca1e8fbb139acc2f2bEli Friedman case Builtin::BImemmove: 6881caae959017b355e9bb61250d5a0d04edbf468b0Daniel Dunbar case Builtin::BI__builtin_memmove: { 689ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman std::pair<llvm::Value*, unsigned> Dest = 690ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman EmitPointerWithAlignment(E->getArg(0)); 691ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman std::pair<llvm::Value*, unsigned> Src = 692ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman EmitPointerWithAlignment(E->getArg(1)); 6933ecd785aff34381f3704d9cb28fe3ef85af759deMon P Wang Value *SizeVal = EmitScalarExpr(E->getArg(2)); 694ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman unsigned Align = std::min(Dest.second, Src.second); 695ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman Builder.CreateMemMove(Dest.first, Src.first, SizeVal, Align, false); 696ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman return RValue::get(Dest.first); 6971caae959017b355e9bb61250d5a0d04edbf468b0Daniel Dunbar } 698e6ec205d6d0f4aec27bf49ca1e8fbb139acc2f2bEli Friedman case Builtin::BImemset: 6991caae959017b355e9bb61250d5a0d04edbf468b0Daniel Dunbar case Builtin::BI__builtin_memset: { 700ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman std::pair<llvm::Value*, unsigned> Dest = 701ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman EmitPointerWithAlignment(E->getArg(0)); 7029f0c7cc36d29cf591c33962931f5862847145f3eBenjamin Kramer Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), 7039f0c7cc36d29cf591c33962931f5862847145f3eBenjamin Kramer Builder.getInt8Ty()); 7043ecd785aff34381f3704d9cb28fe3ef85af759deMon P Wang Value *SizeVal = EmitScalarExpr(E->getArg(2)); 705ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman Builder.CreateMemSet(Dest.first, ByteVal, SizeVal, Dest.second, false); 706ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman return RValue::get(Dest.first); 707d4b32e46517358f34e8cfbea35010adfcc3786e0Eli Friedman } 708a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner case Builtin::BI__builtin___memset_chk: { 709f3477c13eeaf11b32a41f181398fb5deffd0dd73Sylvestre Ledru // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2. 710a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith llvm::APSInt Size, DstSize; 711a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) || 712a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext())) 713a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner break; 714a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner if (Size.ugt(DstSize)) 715a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner break; 716ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman std::pair<llvm::Value*, unsigned> Dest = 717ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman EmitPointerWithAlignment(E->getArg(0)); 718a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), 719a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner Builder.getInt8Ty()); 720a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); 721ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman Builder.CreateMemSet(Dest.first, ByteVal, SizeVal, Dest.second, false); 722ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman return RValue::get(Dest.first); 723a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner } 724fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall case Builtin::BI__builtin_dwarf_cfa: { 725fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall // The offset in bytes from the first argument to the CFA. 726fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall // 727fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall // Why on earth is this in the frontend? Is there any reason at 728fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall // all that the backend can't reasonably determine this while 729fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall // lowering llvm.eh.dwarf.cfa()? 730fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall // 731fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall // TODO: If there's a satisfactory reason, add a target hook for 732fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall // this instead of hard-coding 0, which is correct for most targets. 733fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall int32_t Offset = 0; 734fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall 7358dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa); 736258f930227c1a102c9c22eee88df65f748863425Jim Grosbach return RValue::get(Builder.CreateCall(F, 73777b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner llvm::ConstantInt::get(Int32Ty, Offset))); 738fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall } 739256f77e431bc6b920ec94cf0bb4ad339ca21b8c9Eli Friedman case Builtin::BI__builtin_return_address: { 74083c2a98012a65b51be66fd76c3a1b13ed782c558Anton Korobeynikov Value *Depth = EmitScalarExpr(E->getArg(0)); 741578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer Depth = Builder.CreateIntCast(Depth, Int32Ty, false); 7428dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::returnaddress); 74383c2a98012a65b51be66fd76c3a1b13ed782c558Anton Korobeynikov return RValue::get(Builder.CreateCall(F, Depth)); 744256f77e431bc6b920ec94cf0bb4ad339ca21b8c9Eli Friedman } 745256f77e431bc6b920ec94cf0bb4ad339ca21b8c9Eli Friedman case Builtin::BI__builtin_frame_address: { 74683c2a98012a65b51be66fd76c3a1b13ed782c558Anton Korobeynikov Value *Depth = EmitScalarExpr(E->getArg(0)); 747578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer Depth = Builder.CreateIntCast(Depth, Int32Ty, false); 7488dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::frameaddress); 74983c2a98012a65b51be66fd76c3a1b13ed782c558Anton Korobeynikov return RValue::get(Builder.CreateCall(F, Depth)); 750256f77e431bc6b920ec94cf0bb4ad339ca21b8c9Eli Friedman } 7513b660efb9f9fa3e87096f4a96a2093cd17c43c2eEli Friedman case Builtin::BI__builtin_extract_return_addr: { 752492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall Value *Address = EmitScalarExpr(E->getArg(0)); 753492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall Value *Result = getTargetHooks().decodeReturnAddress(*this, Address); 754492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall return RValue::get(Result); 755492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall } 756492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall case Builtin::BI__builtin_frob_return_addr: { 757492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall Value *Address = EmitScalarExpr(E->getArg(0)); 758492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall Value *Result = getTargetHooks().encodeReturnAddress(*this, Address); 759492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall return RValue::get(Result); 7603b660efb9f9fa3e87096f4a96a2093cd17c43c2eEli Friedman } 7616374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall case Builtin::BI__builtin_dwarf_sp_column: { 7622acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::IntegerType *Ty 7636374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall = cast<llvm::IntegerType>(ConvertType(E->getType())); 7646374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall int Column = getTargetHooks().getDwarfEHStackPointer(CGM); 7656374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall if (Column == -1) { 7666374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall CGM.ErrorUnsupported(E, "__builtin_dwarf_sp_column"); 7676374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall return RValue::get(llvm::UndefValue::get(Ty)); 7686374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall } 7696374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall return RValue::get(llvm::ConstantInt::get(Ty, Column, true)); 7706374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall } 7716374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall case Builtin::BI__builtin_init_dwarf_reg_size_table: { 7726374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall Value *Address = EmitScalarExpr(E->getArg(0)); 7736374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall if (getTargetHooks().initDwarfEHRegSizeTable(*this, Address)) 7746374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall CGM.ErrorUnsupported(E, "__builtin_init_dwarf_reg_size_table"); 7756374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall return RValue::get(llvm::UndefValue::get(ConvertType(E->getType()))); 7766374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall } 7777ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall case Builtin::BI__builtin_eh_return: { 7787ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall Value *Int = EmitScalarExpr(E->getArg(0)); 7797ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall Value *Ptr = EmitScalarExpr(E->getArg(1)); 7807ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall 7812acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType()); 7827ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) && 7837ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall "LLVM's __builtin_eh_return only supports 32- and 64-bit variants"); 7847ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall Value *F = CGM.getIntrinsic(IntTy->getBitWidth() == 32 7857ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall ? Intrinsic::eh_return_i32 7868dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer : Intrinsic::eh_return_i64); 7877ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall Builder.CreateCall2(F, Int, Ptr); 788cd5b22e12b6513163dd131589746c194090f14e6John McCall Builder.CreateUnreachable(); 789cd5b22e12b6513163dd131589746c194090f14e6John McCall 790cd5b22e12b6513163dd131589746c194090f14e6John McCall // We do need to preserve an insertion point. 791d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall EmitBlock(createBasicBlock("builtin_eh_return.cont")); 792cd5b22e12b6513163dd131589746c194090f14e6John McCall 7936bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 7947ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall } 795a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman case Builtin::BI__builtin_unwind_init: { 7968dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init); 797a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman return RValue::get(Builder.CreateCall(F)); 798a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman } 7995e11085830d4d4c53ff75575ab75889ee5126854John McCall case Builtin::BI__builtin_extend_pointer: { 8005e11085830d4d4c53ff75575ab75889ee5126854John McCall // Extends a pointer to the size of an _Unwind_Word, which is 801d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // uint64_t on all platforms. Generally this gets poked into a 802d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // register and eventually used as an address, so if the 803d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // addressing registers are wider than pointers and the platform 804d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // doesn't implicitly ignore high-order bits when doing 805d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // addressing, we need to make sure we zext / sext based on 806d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // the platform's expectations. 8075e11085830d4d4c53ff75575ab75889ee5126854John McCall // 8085e11085830d4d4c53ff75575ab75889ee5126854John McCall // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html 809d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall 810d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // Cast the pointer to intptr_t. 8115e11085830d4d4c53ff75575ab75889ee5126854John McCall Value *Ptr = EmitScalarExpr(E->getArg(0)); 812d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall Value *Result = Builder.CreatePtrToInt(Ptr, IntPtrTy, "extend.cast"); 813d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall 814d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // If that's 64 bits, we're done. 815d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall if (IntPtrTy->getBitWidth() == 64) 816d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall return RValue::get(Result); 817d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall 818d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // Otherwise, ask the codegen data what to do. 819492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall if (getTargetHooks().extendPointerWithSExt()) 820d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall return RValue::get(Builder.CreateSExt(Result, Int64Ty, "extend.sext")); 821d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall else 822d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall return RValue::get(Builder.CreateZExt(Result, Int64Ty, "extend.zext")); 8235e11085830d4d4c53ff75575ab75889ee5126854John McCall } 824a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman case Builtin::BI__builtin_setjmp: { 82578673d9f910e8dfe13248c2426c51d8f9fb28572John McCall // Buffer is a void**. 826a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman Value *Buf = EmitScalarExpr(E->getArg(0)); 82778673d9f910e8dfe13248c2426c51d8f9fb28572John McCall 82878673d9f910e8dfe13248c2426c51d8f9fb28572John McCall // Store the frame pointer to the setjmp buffer. 829a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman Value *FrameAddr = 83078673d9f910e8dfe13248c2426c51d8f9fb28572John McCall Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress), 83177b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner ConstantInt::get(Int32Ty, 0)); 832a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman Builder.CreateStore(FrameAddr, Buf); 83378673d9f910e8dfe13248c2426c51d8f9fb28572John McCall 8346d172e2985346e55095c75f456901ea5d40fddaaJim Grosbach // Store the stack pointer to the setjmp buffer. 8356d172e2985346e55095c75f456901ea5d40fddaaJim Grosbach Value *StackAddr = 8366d172e2985346e55095c75f456901ea5d40fddaaJim Grosbach Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave)); 8376d172e2985346e55095c75f456901ea5d40fddaaJim Grosbach Value *StackSaveSlot = 83877b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner Builder.CreateGEP(Buf, ConstantInt::get(Int32Ty, 2)); 8396d172e2985346e55095c75f456901ea5d40fddaaJim Grosbach Builder.CreateStore(StackAddr, StackSaveSlot); 8406d172e2985346e55095c75f456901ea5d40fddaaJim Grosbach 84178673d9f910e8dfe13248c2426c51d8f9fb28572John McCall // Call LLVM's EH setjmp, which is lightweight. 84278673d9f910e8dfe13248c2426c51d8f9fb28572John McCall Value *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp); 843d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Buf = Builder.CreateBitCast(Buf, Int8PtrTy); 844a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman return RValue::get(Builder.CreateCall(F, Buf)); 845a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman } 846a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman case Builtin::BI__builtin_longjmp: { 847a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman Value *Buf = EmitScalarExpr(E->getArg(0)); 848d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Buf = Builder.CreateBitCast(Buf, Int8PtrTy); 84978673d9f910e8dfe13248c2426c51d8f9fb28572John McCall 85078673d9f910e8dfe13248c2426c51d8f9fb28572John McCall // Call LLVM's EH longjmp, which is lightweight. 85178673d9f910e8dfe13248c2426c51d8f9fb28572John McCall Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf); 85278673d9f910e8dfe13248c2426c51d8f9fb28572John McCall 853cd5b22e12b6513163dd131589746c194090f14e6John McCall // longjmp doesn't return; mark this as unreachable. 854cd5b22e12b6513163dd131589746c194090f14e6John McCall Builder.CreateUnreachable(); 855cd5b22e12b6513163dd131589746c194090f14e6John McCall 856cd5b22e12b6513163dd131589746c194090f14e6John McCall // We do need to preserve an insertion point. 857d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall EmitBlock(createBasicBlock("longjmp.cont")); 858cd5b22e12b6513163dd131589746c194090f14e6John McCall 8596bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 860a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman } 8611ffe281890f3cd7728316b45a1f3dd4d3120af7bMon P Wang case Builtin::BI__sync_fetch_and_add: 8621ffe281890f3cd7728316b45a1f3dd4d3120af7bMon P Wang case Builtin::BI__sync_fetch_and_sub: 8635caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_or: 8645caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_and: 8655caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_xor: 8665caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_add_and_fetch: 8675caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_sub_and_fetch: 8685caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_and_and_fetch: 8695caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_or_and_fetch: 8705caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_xor_and_fetch: 8715caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_val_compare_and_swap: 8725caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_bool_compare_and_swap: 8735caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_test_and_set: 8745caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_release: 87523aa9c8ca0bc441aab2a38a4c9fc7a1c9e9ac16aChris Lattner case Builtin::BI__sync_swap: 876b219cfc4d75f0a03630b7c4509ef791b7e97b2c8David Blaikie llvm_unreachable("Shouldn't make it through sema"); 8775caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_add_1: 8785caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_add_2: 8795caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_add_4: 8805caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_add_8: 8815caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_add_16: 882c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Add, E); 8835caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_sub_1: 8845caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_sub_2: 8855caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_sub_4: 8865caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_sub_8: 8875caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_sub_16: 888c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Sub, E); 8895caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_or_1: 8905caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_or_2: 8915caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_or_4: 8925caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_or_8: 8935caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_or_16: 894c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Or, E); 8955caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_and_1: 8965caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_and_2: 8975caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_and_4: 8985caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_and_8: 8995caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_and_16: 900c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::And, E); 9015caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_xor_1: 9025caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_xor_2: 9035caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_xor_4: 9045caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_xor_8: 9055caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_xor_16: 906c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xor, E); 9071eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 9085caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner // Clang extensions: not overloaded yet. 9091ffe281890f3cd7728316b45a1f3dd4d3120af7bMon P Wang case Builtin::BI__sync_fetch_and_min: 910c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Min, E); 9111ffe281890f3cd7728316b45a1f3dd4d3120af7bMon P Wang case Builtin::BI__sync_fetch_and_max: 912c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Max, E); 9131ffe281890f3cd7728316b45a1f3dd4d3120af7bMon P Wang case Builtin::BI__sync_fetch_and_umin: 914c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMin, E); 9151ffe281890f3cd7728316b45a1f3dd4d3120af7bMon P Wang case Builtin::BI__sync_fetch_and_umax: 916c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMax, E); 9170002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar 9185caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_add_and_fetch_1: 9195caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_add_and_fetch_2: 9205caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_add_and_fetch_4: 9215caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_add_and_fetch_8: 9225caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_add_and_fetch_16: 923c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Add, E, 9240002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar llvm::Instruction::Add); 9255caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_sub_and_fetch_1: 9265caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_sub_and_fetch_2: 9275caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_sub_and_fetch_4: 9285caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_sub_and_fetch_8: 9295caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_sub_and_fetch_16: 930c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Sub, E, 9310002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar llvm::Instruction::Sub); 9325caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_and_and_fetch_1: 9335caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_and_and_fetch_2: 9345caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_and_and_fetch_4: 9355caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_and_and_fetch_8: 9365caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_and_and_fetch_16: 937c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::And, E, 9380002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar llvm::Instruction::And); 9395caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_or_and_fetch_1: 9405caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_or_and_fetch_2: 9415caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_or_and_fetch_4: 9425caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_or_and_fetch_8: 9435caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_or_and_fetch_16: 944c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Or, E, 9450002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar llvm::Instruction::Or); 9465caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_xor_and_fetch_1: 9475caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_xor_and_fetch_2: 9485caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_xor_and_fetch_4: 9495caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_xor_and_fetch_8: 9505caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_xor_and_fetch_16: 951c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Xor, E, 9520002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar llvm::Instruction::Xor); 9531eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 9545caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_val_compare_and_swap_1: 9555caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_val_compare_and_swap_2: 9565caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_val_compare_and_swap_4: 9575caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_val_compare_and_swap_8: 958cb61a7bbe635cfa941b1aeaaa1fbda1bf900ee51Daniel Dunbar case Builtin::BI__sync_val_compare_and_swap_16: { 95926815d97c5743481e317f17a8d53a6819d061862John McCall QualType T = E->getType(); 960d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall llvm::Value *DestPtr = EmitScalarExpr(E->getArg(0)); 961956a5a17713deb1b5b27893303c4f308a1bd2a62Micah Villmow unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); 962258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 9639cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::IntegerType *IntType = 964d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall llvm::IntegerType::get(getLLVMContext(), 965d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall getContext().getTypeSize(T)); 9669cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); 967db4325b098eff5e9e660db19f0148423fb21f27fChandler Carruth 96826815d97c5743481e317f17a8d53a6819d061862John McCall Value *Args[3]; 96926815d97c5743481e317f17a8d53a6819d061862John McCall Args[0] = Builder.CreateBitCast(DestPtr, IntPtrType); 970d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Args[1] = EmitScalarExpr(E->getArg(1)); 9712acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ValueType = Args[1]->getType(); 972d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Args[1] = EmitToInt(*this, Args[1], T, IntType); 973d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Args[2] = EmitToInt(*this, EmitScalarExpr(E->getArg(2)), T, IntType); 97426815d97c5743481e317f17a8d53a6819d061862John McCall 975c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman Value *Result = Builder.CreateAtomicCmpXchg(Args[0], Args[1], Args[2], 976651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::SequentiallyConsistent, 977c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman llvm::SequentiallyConsistent); 978d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Result = EmitFromInt(*this, Result, T, ValueType); 97926815d97c5743481e317f17a8d53a6819d061862John McCall return RValue::get(Result); 980022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson } 9810002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar 9825caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_bool_compare_and_swap_1: 9835caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_bool_compare_and_swap_2: 9845caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_bool_compare_and_swap_4: 9855caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_bool_compare_and_swap_8: 986cb61a7bbe635cfa941b1aeaaa1fbda1bf900ee51Daniel Dunbar case Builtin::BI__sync_bool_compare_and_swap_16: { 98726815d97c5743481e317f17a8d53a6819d061862John McCall QualType T = E->getArg(1)->getType(); 988d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall llvm::Value *DestPtr = EmitScalarExpr(E->getArg(0)); 989956a5a17713deb1b5b27893303c4f308a1bd2a62Micah Villmow unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); 990258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 9919cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::IntegerType *IntType = 992d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall llvm::IntegerType::get(getLLVMContext(), 993d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall getContext().getTypeSize(T)); 9949cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); 995db4325b098eff5e9e660db19f0148423fb21f27fChandler Carruth 99626815d97c5743481e317f17a8d53a6819d061862John McCall Value *Args[3]; 99726815d97c5743481e317f17a8d53a6819d061862John McCall Args[0] = Builder.CreateBitCast(DestPtr, IntPtrType); 998d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Args[1] = EmitToInt(*this, EmitScalarExpr(E->getArg(1)), T, IntType); 999d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Args[2] = EmitToInt(*this, EmitScalarExpr(E->getArg(2)), T, IntType); 100026815d97c5743481e317f17a8d53a6819d061862John McCall 1001db4325b098eff5e9e660db19f0148423fb21f27fChandler Carruth Value *OldVal = Args[1]; 1002c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman Value *PrevVal = Builder.CreateAtomicCmpXchg(Args[0], Args[1], Args[2], 1003651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::SequentiallyConsistent, 1004c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman llvm::SequentiallyConsistent); 10050002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar Value *Result = Builder.CreateICmpEQ(PrevVal, OldVal); 10060002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar // zext bool to int. 100726815d97c5743481e317f17a8d53a6819d061862John McCall Result = Builder.CreateZExt(Result, ConvertType(E->getType())); 100826815d97c5743481e317f17a8d53a6819d061862John McCall return RValue::get(Result); 10090002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar } 10100002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar 101123aa9c8ca0bc441aab2a38a4c9fc7a1c9e9ac16aChris Lattner case Builtin::BI__sync_swap_1: 101223aa9c8ca0bc441aab2a38a4c9fc7a1c9e9ac16aChris Lattner case Builtin::BI__sync_swap_2: 101323aa9c8ca0bc441aab2a38a4c9fc7a1c9e9ac16aChris Lattner case Builtin::BI__sync_swap_4: 101423aa9c8ca0bc441aab2a38a4c9fc7a1c9e9ac16aChris Lattner case Builtin::BI__sync_swap_8: 101523aa9c8ca0bc441aab2a38a4c9fc7a1c9e9ac16aChris Lattner case Builtin::BI__sync_swap_16: 1016c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E); 101723aa9c8ca0bc441aab2a38a4c9fc7a1c9e9ac16aChris Lattner 10185caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_test_and_set_1: 10195caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_test_and_set_2: 10205caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_test_and_set_4: 10215caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_test_and_set_8: 10225caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_test_and_set_16: 1023c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E); 1024cb61a7bbe635cfa941b1aeaaa1fbda1bf900ee51Daniel Dunbar 10255caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_release_1: 10265caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_release_2: 10275caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_release_4: 10285caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_release_8: 1029f58cd9bca9c2e7373300fc8bb7c57cff7e4eda4fChris Lattner case Builtin::BI__sync_lock_release_16: { 1030f58cd9bca9c2e7373300fc8bb7c57cff7e4eda4fChris Lattner Value *Ptr = EmitScalarExpr(E->getArg(0)); 1031eb43f4a8f133c2bc510ae136a556e92b68a6ff44Eli Friedman QualType ElTy = E->getArg(0)->getType()->getPointeeType(); 1032eb43f4a8f133c2bc510ae136a556e92b68a6ff44Eli Friedman CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy); 1033ff993202abf6f5dc41c584c7103f5d39f248b3ddEli Friedman llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(), 1034ff993202abf6f5dc41c584c7103f5d39f248b3ddEli Friedman StoreSize.getQuantity() * 8); 1035ff993202abf6f5dc41c584c7103f5d39f248b3ddEli Friedman Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo()); 1036258f930227c1a102c9c22eee88df65f748863425Jim Grosbach llvm::StoreInst *Store = 1037ff993202abf6f5dc41c584c7103f5d39f248b3ddEli Friedman Builder.CreateStore(llvm::Constant::getNullValue(ITy), Ptr); 1038eb43f4a8f133c2bc510ae136a556e92b68a6ff44Eli Friedman Store->setAlignment(StoreSize.getQuantity()); 1039eb43f4a8f133c2bc510ae136a556e92b68a6ff44Eli Friedman Store->setAtomic(llvm::Release); 10406bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 1041f58cd9bca9c2e7373300fc8bb7c57cff7e4eda4fChris Lattner } 1042ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar 1043f58cd9bca9c2e7373300fc8bb7c57cff7e4eda4fChris Lattner case Builtin::BI__sync_synchronize: { 1044c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman // We assume this is supposed to correspond to a C++0x-style 1045c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman // sequentially-consistent fence (i.e. this is only usable for 1046c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman // synchonization, not device I/O or anything like that). This intrinsic 1047258f930227c1a102c9c22eee88df65f748863425Jim Grosbach // is really badly designed in the sense that in theory, there isn't 1048c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman // any way to safely use it... but in practice, it mostly works 1049c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman // to use it with non-atomic loads and stores to get acquire/release 1050c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman // semantics. 1051c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman Builder.CreateFence(llvm::SequentiallyConsistent); 10526bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 1053f58cd9bca9c2e7373300fc8bb7c57cff7e4eda4fChris Lattner } 10541eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 10552c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case Builtin::BI__c11_atomic_is_lock_free: 10562c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case Builtin::BI__atomic_is_lock_free: { 10572c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith // Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the 10582c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith // __c11 builtin, ptr is 0 (indicating a properly-aligned object), since 10592c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith // _Atomic(T) is always properly-aligned. 10602c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith const char *LibCallName = "__atomic_is_lock_free"; 10612c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith CallArgList Args; 10622c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Args.add(RValue::get(EmitScalarExpr(E->getArg(0))), 10632c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith getContext().getSizeType()); 10642c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith if (BuiltinID == Builtin::BI__atomic_is_lock_free) 10652c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Args.add(RValue::get(EmitScalarExpr(E->getArg(1))), 10662c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith getContext().VoidPtrTy); 10672c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith else 10682c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Args.add(RValue::get(llvm::Constant::getNullValue(VoidPtrTy)), 10692c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith getContext().VoidPtrTy); 10702c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith const CGFunctionInfo &FuncInfo = 10710f3d0970dcdf6cf17550b86838dff12813968dbcJohn McCall CGM.getTypes().arrangeFreeFunctionCall(E->getType(), Args, 10720f3d0970dcdf6cf17550b86838dff12813968dbcJohn McCall FunctionType::ExtInfo(), 10730f3d0970dcdf6cf17550b86838dff12813968dbcJohn McCall RequiredArgs::All); 10742c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo); 10752c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName); 10762c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith return EmitCall(FuncInfo, Func, ReturnValueSlot(), Args); 10772c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 10782c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 10792c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case Builtin::BI__atomic_test_and_set: { 10802c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith // Look at the argument type to determine whether this is a volatile 10812c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith // operation. The parameter type is always volatile. 10822c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType(); 10832c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith bool Volatile = 10842c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified(); 10852c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 10862c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Value *Ptr = EmitScalarExpr(E->getArg(0)); 1087956a5a17713deb1b5b27893303c4f308a1bd2a62Micah Villmow unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace(); 10882c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace)); 10892c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Value *NewVal = Builder.getInt8(1); 10902c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Value *Order = EmitScalarExpr(E->getArg(1)); 10912c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith if (isa<llvm::ConstantInt>(Order)) { 10922c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 10936bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines AtomicRMWInst *Result = nullptr; 10942c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith switch (ord) { 10952c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case 0: // memory_order_relaxed 10962c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith default: // invalid order 10972c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, 10982c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Ptr, NewVal, 10992c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::Monotonic); 11002c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith break; 11012c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case 1: // memory_order_consume 11022c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case 2: // memory_order_acquire 11032c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, 11042c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Ptr, NewVal, 11052c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::Acquire); 11062c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith break; 11072c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case 3: // memory_order_release 11082c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, 11092c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Ptr, NewVal, 11102c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::Release); 11112c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith break; 11122c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case 4: // memory_order_acq_rel 11132c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, 11142c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Ptr, NewVal, 11152c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::AcquireRelease); 11162c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith break; 11172c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case 5: // memory_order_seq_cst 11182c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, 11192c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Ptr, NewVal, 11202c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::SequentiallyConsistent); 11212c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith break; 11222c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 11232c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Result->setVolatile(Volatile); 11242c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith return RValue::get(Builder.CreateIsNotNull(Result, "tobool")); 11252c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 11262c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 11272c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 11282c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 11292c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::BasicBlock *BBs[5] = { 11302c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith createBasicBlock("monotonic", CurFn), 11312c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith createBasicBlock("acquire", CurFn), 11322c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith createBasicBlock("release", CurFn), 11332c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith createBasicBlock("acqrel", CurFn), 11342c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith createBasicBlock("seqcst", CurFn) 11352c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith }; 11362c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::AtomicOrdering Orders[5] = { 11372c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::Monotonic, llvm::Acquire, llvm::Release, 11382c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::AcquireRelease, llvm::SequentiallyConsistent 11392c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith }; 11402c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 11412c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 11422c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]); 11432c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 11442c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Builder.SetInsertPoint(ContBB); 11452c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith PHINode *Result = Builder.CreatePHI(Int8Ty, 5, "was_set"); 11462c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 11472c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith for (unsigned i = 0; i < 5; ++i) { 11482c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Builder.SetInsertPoint(BBs[i]); 11492c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith AtomicRMWInst *RMW = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, 11502c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Ptr, NewVal, Orders[i]); 11512c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith RMW->setVolatile(Volatile); 11522c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Result->addIncoming(RMW, BBs[i]); 11532c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Builder.CreateBr(ContBB); 11542c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 11552c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 11562c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(0), BBs[0]); 11572c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(1), BBs[1]); 11582c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(2), BBs[1]); 11592c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(3), BBs[2]); 11602c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(4), BBs[3]); 11612c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(5), BBs[4]); 11622c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 11632c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Builder.SetInsertPoint(ContBB); 11642c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith return RValue::get(Builder.CreateIsNotNull(Result, "tobool")); 11652c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 11662c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 11672c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case Builtin::BI__atomic_clear: { 11682c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType(); 11692c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith bool Volatile = 11702c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified(); 11712c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 11722c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Value *Ptr = EmitScalarExpr(E->getArg(0)); 1173956a5a17713deb1b5b27893303c4f308a1bd2a62Micah Villmow unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace(); 11742c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace)); 11752c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Value *NewVal = Builder.getInt8(0); 11762c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Value *Order = EmitScalarExpr(E->getArg(1)); 11772c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith if (isa<llvm::ConstantInt>(Order)) { 11782c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 11792c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile); 11802c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Store->setAlignment(1); 11812c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith switch (ord) { 11822c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case 0: // memory_order_relaxed 11832c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith default: // invalid order 11842c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Store->setOrdering(llvm::Monotonic); 11852c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith break; 11862c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case 3: // memory_order_release 11872c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Store->setOrdering(llvm::Release); 11882c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith break; 11892c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case 5: // memory_order_seq_cst 11902c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Store->setOrdering(llvm::SequentiallyConsistent); 11912c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith break; 11922c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 11936bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 11942c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 11952c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 11962c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 11972c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 11982c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::BasicBlock *BBs[3] = { 11992c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith createBasicBlock("monotonic", CurFn), 12002c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith createBasicBlock("release", CurFn), 12012c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith createBasicBlock("seqcst", CurFn) 12022c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith }; 12032c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::AtomicOrdering Orders[3] = { 12042c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::Monotonic, llvm::Release, llvm::SequentiallyConsistent 12052c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith }; 12062c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 12072c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 12082c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]); 12092c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 12102c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith for (unsigned i = 0; i < 3; ++i) { 12112c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Builder.SetInsertPoint(BBs[i]); 12122c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile); 12132c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Store->setAlignment(1); 12142c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Store->setOrdering(Orders[i]); 12152c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Builder.CreateBr(ContBB); 12162c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 12172c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 12182c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(0), BBs[0]); 12192c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(3), BBs[1]); 12202c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(5), BBs[2]); 12212c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 12222c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Builder.SetInsertPoint(ContBB); 12236bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 12242c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 12252c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 1226276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman case Builtin::BI__atomic_thread_fence: 1227fafbf06732746f3ceca21d452d77b144ba8652aeRichard Smith case Builtin::BI__atomic_signal_fence: 1228fafbf06732746f3ceca21d452d77b144ba8652aeRichard Smith case Builtin::BI__c11_atomic_thread_fence: 1229fafbf06732746f3ceca21d452d77b144ba8652aeRichard Smith case Builtin::BI__c11_atomic_signal_fence: { 1230276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman llvm::SynchronizationScope Scope; 1231fafbf06732746f3ceca21d452d77b144ba8652aeRichard Smith if (BuiltinID == Builtin::BI__atomic_signal_fence || 1232fafbf06732746f3ceca21d452d77b144ba8652aeRichard Smith BuiltinID == Builtin::BI__c11_atomic_signal_fence) 1233276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Scope = llvm::SingleThread; 1234276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman else 1235276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Scope = llvm::CrossThread; 1236276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Value *Order = EmitScalarExpr(E->getArg(0)); 1237276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman if (isa<llvm::ConstantInt>(Order)) { 1238276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1239276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman switch (ord) { 1240276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman case 0: // memory_order_relaxed 1241276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman default: // invalid order 1242276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman break; 1243276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman case 1: // memory_order_consume 1244276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman case 2: // memory_order_acquire 1245276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateFence(llvm::Acquire, Scope); 1246276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman break; 1247276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman case 3: // memory_order_release 1248276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateFence(llvm::Release, Scope); 1249276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman break; 1250276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman case 4: // memory_order_acq_rel 1251276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateFence(llvm::AcquireRelease, Scope); 1252276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman break; 1253276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman case 5: // memory_order_seq_cst 1254276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateFence(llvm::SequentiallyConsistent, Scope); 1255276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman break; 1256276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman } 12576bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 1258276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman } 1259276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman 1260276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB; 1261276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman AcquireBB = createBasicBlock("acquire", CurFn); 1262276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman ReleaseBB = createBasicBlock("release", CurFn); 1263276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman AcqRelBB = createBasicBlock("acqrel", CurFn); 1264276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman SeqCstBB = createBasicBlock("seqcst", CurFn); 1265276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1266276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman 1267276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1268276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman llvm::SwitchInst *SI = Builder.CreateSwitch(Order, ContBB); 1269276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman 1270276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.SetInsertPoint(AcquireBB); 1271276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateFence(llvm::Acquire, Scope); 1272276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateBr(ContBB); 1273276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman SI->addCase(Builder.getInt32(1), AcquireBB); 1274276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman SI->addCase(Builder.getInt32(2), AcquireBB); 1275276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman 1276276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.SetInsertPoint(ReleaseBB); 1277276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateFence(llvm::Release, Scope); 1278276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateBr(ContBB); 1279276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman SI->addCase(Builder.getInt32(3), ReleaseBB); 1280276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman 1281276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.SetInsertPoint(AcqRelBB); 1282276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateFence(llvm::AcquireRelease, Scope); 1283276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateBr(ContBB); 1284276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman SI->addCase(Builder.getInt32(4), AcqRelBB); 1285276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman 1286276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.SetInsertPoint(SeqCstBB); 1287276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateFence(llvm::SequentiallyConsistent, Scope); 1288276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateBr(ContBB); 1289276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman SI->addCase(Builder.getInt32(5), SeqCstBB); 1290276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman 1291276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.SetInsertPoint(ContBB); 12926bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 1293276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman } 1294276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman 1295ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar // Library functions with special handling. 1296ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar case Builtin::BIsqrt: 1297ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar case Builtin::BIsqrtf: 1298ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar case Builtin::BIsqrtl: { 1299ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel // Transform a call to sqrt* into a @llvm.sqrt.* intrinsic call, but only 1300ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel // in finite- or unsafe-math mode (the intrinsic has different semantics 1301ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel // for handling negative numbers compared to the library function, so 1302ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel // -fmath-errno=0 is not enough). 1303ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel if (!FD->hasAttr<ConstAttr>()) 1304ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel break; 1305ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel if (!(CGM.getCodeGenOpts().UnsafeFPMath || 1306ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel CGM.getCodeGenOpts().NoNaNsFPMath)) 1307ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel break; 1308ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel Value *Arg0 = EmitScalarExpr(E->getArg(0)); 1309ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel llvm::Type *ArgType = Arg0->getType(); 1310ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel Value *F = CGM.getIntrinsic(Intrinsic::sqrt, ArgType); 1311ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel return RValue::get(Builder.CreateCall(F, Arg0)); 1312ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar } 1313ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar 1314ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar case Builtin::BIpow: 1315ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar case Builtin::BIpowf: 1316ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar case Builtin::BIpowl: { 13170323a7887b26f727080d2aa38f1a325dd23a1ee9Eli Bendersky // Transform a call to pow* into a @llvm.pow.* intrinsic call. 13180323a7887b26f727080d2aa38f1a325dd23a1ee9Eli Bendersky if (!FD->hasAttr<ConstAttr>()) 13190323a7887b26f727080d2aa38f1a325dd23a1ee9Eli Bendersky break; 13200323a7887b26f727080d2aa38f1a325dd23a1ee9Eli Bendersky Value *Base = EmitScalarExpr(E->getArg(0)); 13210323a7887b26f727080d2aa38f1a325dd23a1ee9Eli Bendersky Value *Exponent = EmitScalarExpr(E->getArg(1)); 13220323a7887b26f727080d2aa38f1a325dd23a1ee9Eli Bendersky llvm::Type *ArgType = Base->getType(); 13230323a7887b26f727080d2aa38f1a325dd23a1ee9Eli Bendersky Value *F = CGM.getIntrinsic(Intrinsic::pow, ArgType); 13240323a7887b26f727080d2aa38f1a325dd23a1ee9Eli Bendersky return RValue::get(Builder.CreateCall2(F, Base, Exponent)); 1325ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar } 1326ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman 1327094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich case Builtin::BIfma: 1328094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich case Builtin::BIfmaf: 1329094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich case Builtin::BIfmal: 1330094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich case Builtin::BI__builtin_fma: 1331094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich case Builtin::BI__builtin_fmaf: 1332094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich case Builtin::BI__builtin_fmal: { 1333094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich // Rewrite fma to intrinsic. 1334094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich Value *FirstArg = EmitScalarExpr(E->getArg(0)); 13359cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *ArgType = FirstArg->getType(); 13368dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::fma, ArgType); 1337094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich return RValue::get(Builder.CreateCall3(F, FirstArg, 1338094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich EmitScalarExpr(E->getArg(1)), 1339578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer EmitScalarExpr(E->getArg(2)))); 1340094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich } 1341094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich 1342ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman case Builtin::BI__builtin_signbit: 1343ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman case Builtin::BI__builtin_signbitf: 1344ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman case Builtin::BI__builtin_signbitl: { 1345ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman LLVMContext &C = CGM.getLLVMContext(); 1346ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman 1347ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman Value *Arg = EmitScalarExpr(E->getArg(0)); 13482acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ArgTy = Arg->getType(); 1349ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman if (ArgTy->isPPC_FP128Ty()) 1350ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman break; // FIXME: I'm not sure what the right implementation is here. 1351ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman int ArgWidth = ArgTy->getPrimitiveSizeInBits(); 13522acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ArgIntTy = llvm::IntegerType::get(C, ArgWidth); 1353ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman Value *BCArg = Builder.CreateBitCast(Arg, ArgIntTy); 1354ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman Value *ZeroCmp = llvm::Constant::getNullValue(ArgIntTy); 1355ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman Value *Result = Builder.CreateICmpSLT(BCArg, ZeroCmp); 1356ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman return RValue::get(Builder.CreateZExt(Result, ConvertType(E->getType()))); 1357ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman } 135877f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge case Builtin::BI__builtin_annotation: { 135977f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge llvm::Value *AnnVal = EmitScalarExpr(E->getArg(0)); 136077f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::annotation, 136177f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge AnnVal->getType()); 136277f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge 136377f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge // Get the annotation string, go through casts. Sema requires this to be a 136477f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge // non-wide string literal, potentially casted, so the cast<> is safe. 136577f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge const Expr *AnnotationStrExpr = E->getArg(1)->IgnoreParenCasts(); 1366cfa88f893915ceb8ae4ce2f17c46c24a4d67502fDmitri Gribenko StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString(); 136777f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge return RValue::get(EmitAnnotationCall(F, AnnVal, Str, E->getExprLoc())); 136877f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge } 1369ee76e7215d27c25bb053b71ae0a59119c08ccef9Michael Gottesman case Builtin::BI__builtin_addcb: 13700cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman case Builtin::BI__builtin_addcs: 13710cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman case Builtin::BI__builtin_addc: 13720cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman case Builtin::BI__builtin_addcl: 13737c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_addcll: 1374ee76e7215d27c25bb053b71ae0a59119c08ccef9Michael Gottesman case Builtin::BI__builtin_subcb: 13757c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_subcs: 13767c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_subc: 13777c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_subcl: 13787c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_subcll: { 13790cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman 13800cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // We translate all of these builtins from expressions of the form: 13810cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // int x = ..., y = ..., carryin = ..., carryout, result; 13820cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // result = __builtin_addc(x, y, carryin, &carryout); 13830cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // 13840cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // to LLVM IR of the form: 13850cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // 13860cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // %tmp1 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %x, i32 %y) 13870cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // %tmpsum1 = extractvalue {i32, i1} %tmp1, 0 13880cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // %carry1 = extractvalue {i32, i1} %tmp1, 1 13890cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // %tmp2 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %tmpsum1, 13900cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // i32 %carryin) 13910cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // %result = extractvalue {i32, i1} %tmp2, 0 13920cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // %carry2 = extractvalue {i32, i1} %tmp2, 1 13930cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // %tmp3 = or i1 %carry1, %carry2 13940cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // %tmp4 = zext i1 %tmp3 to i32 13950cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // store i32 %tmp4, i32* %carryout 13960cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman 13970cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // Scalarize our inputs. 13980cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *X = EmitScalarExpr(E->getArg(0)); 13990cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *Y = EmitScalarExpr(E->getArg(1)); 14000cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *Carryin = EmitScalarExpr(E->getArg(2)); 14010cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman std::pair<llvm::Value*, unsigned> CarryOutPtr = 14020cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman EmitPointerWithAlignment(E->getArg(3)); 14030cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman 14047c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman // Decide if we are lowering to a uadd.with.overflow or usub.with.overflow. 14057c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman llvm::Intrinsic::ID IntrinsicId; 14067c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman switch (BuiltinID) { 14077c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman default: llvm_unreachable("Unknown multiprecision builtin id."); 1408ee76e7215d27c25bb053b71ae0a59119c08ccef9Michael Gottesman case Builtin::BI__builtin_addcb: 14097c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_addcs: 14107c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_addc: 14117c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_addcl: 14127c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_addcll: 14137c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman IntrinsicId = llvm::Intrinsic::uadd_with_overflow; 14147c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman break; 1415ee76e7215d27c25bb053b71ae0a59119c08ccef9Michael Gottesman case Builtin::BI__builtin_subcb: 14167c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_subcs: 14177c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_subc: 14187c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_subcl: 14197c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_subcll: 14207c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman IntrinsicId = llvm::Intrinsic::usub_with_overflow; 14217c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman break; 14227c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman } 14230cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman 14240cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // Construct our resulting LLVM IR expression. 14250cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *Carry1; 14260cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *Sum1 = EmitOverflowIntrinsic(*this, IntrinsicId, 14270cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman X, Y, Carry1); 14280cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *Carry2; 14290cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *Sum2 = EmitOverflowIntrinsic(*this, IntrinsicId, 14300cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman Sum1, Carryin, Carry2); 14310cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *CarryOut = Builder.CreateZExt(Builder.CreateOr(Carry1, Carry2), 14320cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman X->getType()); 14330cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::StoreInst *CarryOutStore = Builder.CreateStore(CarryOut, 14340cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman CarryOutPtr.first); 14350cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman CarryOutStore->setAlignment(CarryOutPtr.second); 14360cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman return RValue::get(Sum2); 14370cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman } 143898d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_uadd_overflow: 143998d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_uaddl_overflow: 144098d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_uaddll_overflow: 144198d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_usub_overflow: 144298d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_usubl_overflow: 144398d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_usubll_overflow: 144498d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_umul_overflow: 144598d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_umull_overflow: 144698d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_umulll_overflow: 144798d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_sadd_overflow: 144898d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_saddl_overflow: 144998d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_saddll_overflow: 145098d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_ssub_overflow: 145198d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_ssubl_overflow: 145298d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_ssubll_overflow: 145398d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_smul_overflow: 145498d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_smull_overflow: 145598d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_smulll_overflow: { 145698d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman 145798d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman // We translate all of these builtins directly to the relevant llvm IR node. 145898d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman 145998d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman // Scalarize our inputs. 146098d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman llvm::Value *X = EmitScalarExpr(E->getArg(0)); 146198d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman llvm::Value *Y = EmitScalarExpr(E->getArg(1)); 146298d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman std::pair<llvm::Value *, unsigned> SumOutPtr = 146398d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman EmitPointerWithAlignment(E->getArg(2)); 146498d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman 146598d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman // Decide which of the overflow intrinsics we are lowering to: 146698d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman llvm::Intrinsic::ID IntrinsicId; 146798d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman switch (BuiltinID) { 146898d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman default: llvm_unreachable("Unknown security overflow builtin id."); 146998d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_uadd_overflow: 147098d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_uaddl_overflow: 147198d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_uaddll_overflow: 147298d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman IntrinsicId = llvm::Intrinsic::uadd_with_overflow; 147398d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman break; 147498d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_usub_overflow: 147598d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_usubl_overflow: 147698d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_usubll_overflow: 147798d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman IntrinsicId = llvm::Intrinsic::usub_with_overflow; 147898d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman break; 147998d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_umul_overflow: 148098d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_umull_overflow: 148198d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_umulll_overflow: 148298d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman IntrinsicId = llvm::Intrinsic::umul_with_overflow; 148398d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman break; 148498d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_sadd_overflow: 148598d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_saddl_overflow: 148698d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_saddll_overflow: 148798d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman IntrinsicId = llvm::Intrinsic::sadd_with_overflow; 148898d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman break; 148998d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_ssub_overflow: 149098d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_ssubl_overflow: 149198d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_ssubll_overflow: 149298d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman IntrinsicId = llvm::Intrinsic::ssub_with_overflow; 149398d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman break; 149498d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_smul_overflow: 149598d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_smull_overflow: 149698d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_smulll_overflow: 149798d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman IntrinsicId = llvm::Intrinsic::smul_with_overflow; 149898d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman break; 149998d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman } 150098d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman 150198d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman 150298d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman llvm::Value *Carry; 150398d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman llvm::Value *Sum = EmitOverflowIntrinsic(*this, IntrinsicId, X, Y, Carry); 150498d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman llvm::StoreInst *SumOutStore = Builder.CreateStore(Sum, SumOutPtr.first); 150598d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman SumOutStore->setAlignment(SumOutPtr.second); 150698d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman 150798d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman return RValue::get(Carry); 150898d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman } 15095154dce6388e3aaa445467030df7a45ed1211abeRichard Smith case Builtin::BI__builtin_addressof: 15105154dce6388e3aaa445467030df7a45ed1211abeRichard Smith return RValue::get(EmitLValue(E->getArg(0)).getAddress()); 151127844533aad5c165523a5926424097699610d3f0Nico Weber case Builtin::BI__noop: 15126bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 1513651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case Builtin::BI_InterlockedCompareExchange: { 1514651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AtomicCmpXchgInst *CXI = Builder.CreateAtomicCmpXchg( 1515651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines EmitScalarExpr(E->getArg(0)), 1516651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines EmitScalarExpr(E->getArg(2)), 1517651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines EmitScalarExpr(E->getArg(1)), 1518651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SequentiallyConsistent, 1519651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SequentiallyConsistent); 1520651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CXI->setVolatile(true); 1521651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return RValue::get(CXI); 1522651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 1523651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case Builtin::BI_InterlockedIncrement: { 1524651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( 1525651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AtomicRMWInst::Add, 1526651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines EmitScalarExpr(E->getArg(0)), 1527651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ConstantInt::get(Int32Ty, 1), 1528651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::SequentiallyConsistent); 1529651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines RMWI->setVolatile(true); 1530651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return RValue::get(Builder.CreateAdd(RMWI, ConstantInt::get(Int32Ty, 1))); 1531651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 1532651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case Builtin::BI_InterlockedDecrement: { 1533651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( 1534651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AtomicRMWInst::Sub, 1535651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines EmitScalarExpr(E->getArg(0)), 1536651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ConstantInt::get(Int32Ty, 1), 1537651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::SequentiallyConsistent); 1538651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines RMWI->setVolatile(true); 1539651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return RValue::get(Builder.CreateSub(RMWI, ConstantInt::get(Int32Ty, 1))); 1540651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 1541651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case Builtin::BI_InterlockedExchangeAdd: { 1542651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( 1543651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AtomicRMWInst::Add, 1544651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines EmitScalarExpr(E->getArg(0)), 1545651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines EmitScalarExpr(E->getArg(1)), 1546651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::SequentiallyConsistent); 1547651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines RMWI->setVolatile(true); 1548651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return RValue::get(RMWI); 1549651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 15507ea2e3f6aae9b7511686d3d26dee690fee81c3aaNate Begeman } 15511eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1552a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // If this is an alias for a lib function (e.g. __builtin_sin), emit 1553a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // the call using the normal call path, but using the unmangled 1554a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // version of the function name. 1555a45680b7e7c49ea9893c6cff585984f3e4120366John McCall if (getContext().BuiltinInfo.isLibFunction(BuiltinID)) 1556a45680b7e7c49ea9893c6cff585984f3e4120366John McCall return emitLibraryCall(*this, FD, E, 1557a45680b7e7c49ea9893c6cff585984f3e4120366John McCall CGM.getBuiltinLibFunction(FD, BuiltinID)); 1558258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 1559a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // If this is a predefined lib function (e.g. malloc), emit the call 1560a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // using exactly the normal call path. 1561a45680b7e7c49ea9893c6cff585984f3e4120366John McCall if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID)) 1562a45680b7e7c49ea9893c6cff585984f3e4120366John McCall return emitLibraryCall(*this, FD, E, EmitScalarExpr(E->getCallee())); 15631eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1564b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner // See if we have a target specific intrinsic. 1565a6f80ef997f0363386749087b325607eaa5adcfcDale Johannesen const char *Name = getContext().BuiltinInfo.GetName(BuiltinID); 156655cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic; 156755cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar if (const char *Prefix = 156864aa4b3ec7e62288e2e66c1935487ece995ca94bJohn McCall llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch())) 156955cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar IntrinsicID = Intrinsic::getIntrinsicForGCCBuiltin(Prefix, Name); 15701eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1571b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner if (IntrinsicID != Intrinsic::not_intrinsic) { 1572b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner SmallVector<Value*, 16> Args; 15731eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 157446c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner // Find out if any arguments are required to be integer constant 157546c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner // expressions. 157646c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner unsigned ICEArguments = 0; 157746c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner ASTContext::GetBuiltinTypeError Error; 157846c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); 157946c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner assert(Error == ASTContext::GE_None && "Should not codegen an error"); 158046c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner 1581b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner Function *F = CGM.getIntrinsic(IntrinsicID); 15822acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::FunctionType *FTy = F->getFunctionType(); 15831eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1584b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) { 158546c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner Value *ArgValue; 158646c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner // If this is a normal argument, just emit it as a scalar. 158746c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner if ((ICEArguments & (1 << i)) == 0) { 158846c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner ArgValue = EmitScalarExpr(E->getArg(i)); 158946c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner } else { 1590258f930227c1a102c9c22eee88df65f748863425Jim Grosbach // If this is required to be a constant, constant fold it so that we 159146c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner // know that the generated intrinsic gets a ConstantInt. 159246c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner llvm::APSInt Result; 159346c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result,getContext()); 159446c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner assert(IsConst && "Constant arg isn't actually constant?"); 159546c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner (void)IsConst; 1596d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall ArgValue = llvm::ConstantInt::get(getLLVMContext(), Result); 159746c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner } 15981eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1599b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner // If the intrinsic arg type is different from the builtin arg type 1600b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner // we need to do a bit cast. 16012acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *PTy = FTy->getParamType(i); 1602b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner if (PTy != ArgValue->getType()) { 1603b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner assert(PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) && 1604b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner "Must be able to losslessly bit cast to param"); 1605b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner ArgValue = Builder.CreateBitCast(ArgValue, PTy); 1606b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner } 16071eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1608b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner Args.push_back(ArgValue); 1609b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner } 16101eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 16114c7d9f1507d0f102bd4133bba63348636facd469Jay Foad Value *V = Builder.CreateCall(F, Args); 1612b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner QualType BuiltinRetType = E->getType(); 16131eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 16148b418685e9e4f02f4eb2a76e1ec063e07552b68dChris Lattner llvm::Type *RetTy = VoidTy; 1615258f930227c1a102c9c22eee88df65f748863425Jim Grosbach if (!BuiltinRetType->isVoidType()) 16168b418685e9e4f02f4eb2a76e1ec063e07552b68dChris Lattner RetTy = ConvertType(BuiltinRetType); 16171eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1618b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner if (RetTy != V->getType()) { 1619b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner assert(V->getType()->canLosslesslyBitCastTo(RetTy) && 1620b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner "Must be able to losslessly bit cast result type"); 1621b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner V = Builder.CreateBitCast(V, RetTy); 1622b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner } 16231eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1624b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner return RValue::get(V); 1625b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner } 16261eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1627b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner // See if we have a target specific builtin that needs to be lowered. 1628f02e9ddf5efc75917af712b3c7f909581205f0a5Daniel Dunbar if (Value *V = EmitTargetBuiltinExpr(BuiltinID, E)) 1629b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner return RValue::get(V); 16301eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1631488e993a135ce700b982bf099c3d6b856301d642Daniel Dunbar ErrorUnsupported(E, "builtin function"); 16321eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1633b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner // Unknown builtin, for now just dump it out and return undef. 16349d232c884ea9872d6555df0fd7359699819bc1f1John McCall return GetUndefRValue(E->getType()); 16351eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump} 1636564f1de67d7ba43646b8740db86d6269e3dfbe0bAnders Carlsson 1637f02e9ddf5efc75917af712b3c7f909581205f0a5Daniel DunbarValue *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID, 1638f02e9ddf5efc75917af712b3c7f909581205f0a5Daniel Dunbar const CallExpr *E) { 163964aa4b3ec7e62288e2e66c1935487ece995ca94bJohn McCall switch (getTarget().getTriple().getArch()) { 16402752c0137d95aa2f4ee1cdff4b564bac842e041bChris Lattner case llvm::Triple::arm: 1641651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case llvm::Triple::armeb: 16422752c0137d95aa2f4ee1cdff4b564bac842e041bChris Lattner case llvm::Triple::thumb: 1643651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case llvm::Triple::thumbeb: 16442752c0137d95aa2f4ee1cdff4b564bac842e041bChris Lattner return EmitARMBuiltinExpr(BuiltinID, E); 16456bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case llvm::Triple::aarch64: 16466bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case llvm::Triple::aarch64_be: 1647651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case llvm::Triple::arm64: 16486bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case llvm::Triple::arm64_be: 16496bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitAArch64BuiltinExpr(BuiltinID, E); 165055cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar case llvm::Triple::x86: 165155cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar case llvm::Triple::x86_64: 1652f02e9ddf5efc75917af712b3c7f909581205f0a5Daniel Dunbar return EmitX86BuiltinExpr(BuiltinID, E); 165355cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar case llvm::Triple::ppc: 165455cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar case llvm::Triple::ppc64: 1655ea7fb0ce25acc04664a2e7c2b24af03cef2c0d1fBill Schmidt case llvm::Triple::ppc64le: 1656f02e9ddf5efc75917af712b3c7f909581205f0a5Daniel Dunbar return EmitPPCBuiltinExpr(BuiltinID, E); 165755cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar default: 16586bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 165955cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar } 1660f02e9ddf5efc75917af712b3c7f909581205f0a5Daniel Dunbar} 1661f02e9ddf5efc75917af712b3c7f909581205f0a5Daniel Dunbar 16628b418685e9e4f02f4eb2a76e1ec063e07552b68dChris Lattnerstatic llvm::VectorType *GetNeonType(CodeGenFunction *CGF, 1663aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu NeonTypeFlags TypeFlags, 1664aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu bool V1Ty=false) { 166583084c863572b48579767a4dd5dc686e1a8d669dNAKAMURA Takumi int IsQuad = TypeFlags.isQuad(); 166683084c863572b48579767a4dd5dc686e1a8d669dNAKAMURA Takumi switch (TypeFlags.getEltType()) { 1667da95f73b59f9af964e33725c515139d34c90c863Bob Wilson case NeonTypeFlags::Int8: 1668da95f73b59f9af964e33725c515139d34c90c863Bob Wilson case NeonTypeFlags::Poly8: 1669aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu return llvm::VectorType::get(CGF->Int8Ty, V1Ty ? 1 : (8 << IsQuad)); 1670da95f73b59f9af964e33725c515139d34c90c863Bob Wilson case NeonTypeFlags::Int16: 1671da95f73b59f9af964e33725c515139d34c90c863Bob Wilson case NeonTypeFlags::Poly16: 1672da95f73b59f9af964e33725c515139d34c90c863Bob Wilson case NeonTypeFlags::Float16: 1673aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu return llvm::VectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad)); 1674da95f73b59f9af964e33725c515139d34c90c863Bob Wilson case NeonTypeFlags::Int32: 1675aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu return llvm::VectorType::get(CGF->Int32Ty, V1Ty ? 1 : (2 << IsQuad)); 1676da95f73b59f9af964e33725c515139d34c90c863Bob Wilson case NeonTypeFlags::Int64: 1677624bb5e59dbcc24efeee7dff12c9b48d2b5077e9Kevin Qin case NeonTypeFlags::Poly64: 1678aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu return llvm::VectorType::get(CGF->Int64Ty, V1Ty ? 1 : (1 << IsQuad)); 1679651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NeonTypeFlags::Poly128: 1680651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: i128 and f128 doesn't get fully support in Clang and llvm. 1681651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // There is a lot of i128 and f128 API missing. 1682651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // so we use v16i8 to represent poly128 and get pattern matched. 1683651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return llvm::VectorType::get(CGF->Int8Ty, 16); 1684da95f73b59f9af964e33725c515139d34c90c863Bob Wilson case NeonTypeFlags::Float32: 1685aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu return llvm::VectorType::get(CGF->FloatTy, V1Ty ? 1 : (2 << IsQuad)); 1686b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover case NeonTypeFlags::Float64: 1687aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu return llvm::VectorType::get(CGF->DoubleTy, V1Ty ? 1 : (1 << IsQuad)); 1688561d3abc881033776ece385a01a510e1cbc1fa92David Blaikie } 16890cd6bd62f3f7ee42f08ad130395ac65564768990Benjamin Kramer llvm_unreachable("Unknown vector element type!"); 1690998622c10198a25ba06c93d7e908f88ba0acc920Nate Begeman} 1691998622c10198a25ba06c93d7e908f88ba0acc920Nate Begeman 1692cf55652cf668c1402eee0b12edd2e5a1bc34d7a1Bob WilsonValue *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C) { 1693d075c01c359b9cc120c3accc7166990f9f4ac423Nate Begeman unsigned nElts = cast<llvm::VectorType>(V->getType())->getNumElements(); 16942ce8842641cc312628c4be836d34eb250e7c3f78Chris Lattner Value* SV = llvm::ConstantVector::getSplat(nElts, C); 1695d075c01c359b9cc120c3accc7166990f9f4ac423Nate Begeman return Builder.CreateShuffleVector(V, V, SV, "lane"); 1696d075c01c359b9cc120c3accc7166990f9f4ac423Nate Begeman} 1697d075c01c359b9cc120c3accc7166990f9f4ac423Nate Begeman 169830d91718a676177f0d0d0210ce4fdb4f616df6e5Nate BegemanValue *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops, 1699db3d4d036037f379f12643e067b229862d61e932Bob Wilson const char *name, 170061eecf5aea018ea65c9ab0bccacd2996b15c632dNate Begeman unsigned shift, bool rightshift) { 170130d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman unsigned j = 0; 170230d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end(); 170330d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman ai != ae; ++ai, ++j) 170461eecf5aea018ea65c9ab0bccacd2996b15c632dNate Begeman if (shift > 0 && shift == j) 170561eecf5aea018ea65c9ab0bccacd2996b15c632dNate Begeman Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift); 170661eecf5aea018ea65c9ab0bccacd2996b15c632dNate Begeman else 170761eecf5aea018ea65c9ab0bccacd2996b15c632dNate Begeman Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name); 170830d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman 17094c7d9f1507d0f102bd4133bba63348636facd469Jay Foad return Builder.CreateCall(F, Ops, name); 171030d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman} 171130d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman 1712258f930227c1a102c9c22eee88df65f748863425Jim GrosbachValue *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty, 1713464ccb68f22a7e1c0a2844551c16f721540c91c3Nate Begeman bool neg) { 17142ce8842641cc312628c4be836d34eb250e7c3f78Chris Lattner int SV = cast<ConstantInt>(V)->getSExtValue(); 1715258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 17162acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::VectorType *VTy = cast<llvm::VectorType>(Ty); 1717464ccb68f22a7e1c0a2844551c16f721540c91c3Nate Begeman llvm::Constant *C = ConstantInt::get(VTy->getElementType(), neg ? -SV : SV); 17182ce8842641cc312628c4be836d34eb250e7c3f78Chris Lattner return llvm::ConstantVector::getSplat(VTy->getNumElements(), C); 1719464ccb68f22a7e1c0a2844551c16f721540c91c3Nate Begeman} 1720464ccb68f22a7e1c0a2844551c16f721540c91c3Nate Begeman 17217f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville// \brief Right-shift a vector by a constant. 17227f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la VieuvilleValue *CodeGenFunction::EmitNeonRShiftImm(Value *Vec, Value *Shift, 17237f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville llvm::Type *Ty, bool usgn, 17247f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville const char *name) { 17257f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville llvm::VectorType *VTy = cast<llvm::VectorType>(Ty); 17267f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville 17277f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville int ShiftAmt = cast<ConstantInt>(Shift)->getSExtValue(); 17287f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville int EltSize = VTy->getScalarSizeInBits(); 17297f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville 17307f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville Vec = Builder.CreateBitCast(Vec, Ty); 17317f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville 17327f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville // lshr/ashr are undefined when the shift amount is equal to the vector 17337f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville // element size. 17347f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville if (ShiftAmt == EltSize) { 17357f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville if (usgn) { 17367f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville // Right-shifting an unsigned value by its size yields 0. 17377f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville llvm::Constant *Zero = ConstantInt::get(VTy->getElementType(), 0); 17387f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville return llvm::ConstantVector::getSplat(VTy->getNumElements(), Zero); 17397f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville } else { 17407f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville // Right-shifting a signed value by its size is equivalent 17417f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville // to a shift of size-1. 17427f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville --ShiftAmt; 17437f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt); 17447f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville } 17457f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville } 17467f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville 17477f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville Shift = EmitNeonShiftVector(Shift, Ty, false); 17487f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville if (usgn) 17497f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville return Builder.CreateLShr(Vec, Shift, name); 17507f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville else 17517f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville return Builder.CreateAShr(Vec, Shift, name); 17527f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville} 17537f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville 1754651f13cea278ec967336033dd032faef0e9fc2ecStephen HinesValue *CodeGenFunction::EmitConcatVectors(Value *Lo, Value *Hi, 1755651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *ArgTy) { 1756651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned NumElts = ArgTy->getVectorNumElements(); 1757651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Constant *, 16> Indices; 1758651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines for (unsigned i = 0; i < 2 * NumElts; ++i) 1759651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Indices.push_back(ConstantInt::get(Int32Ty, i)); 1760651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1761651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Constant *Mask = ConstantVector::get(Indices); 1762651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *LoCast = Builder.CreateBitCast(Lo, ArgTy); 1763651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *HiCast = Builder.CreateBitCast(Hi, ArgTy); 1764651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateShuffleVector(LoCast, HiCast, Mask, "concat"); 1765651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 1766651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1767651f13cea278ec967336033dd032faef0e9fc2ecStephen HinesValue *CodeGenFunction::EmitExtractHigh(Value *Vec, llvm::Type *ResTy) { 1768651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned NumElts = ResTy->getVectorNumElements(); 1769651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Constant *, 8> Indices; 1770651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1771651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *InTy = llvm::VectorType::get(ResTy->getVectorElementType(), 1772651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NumElts * 2); 1773651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *VecCast = Builder.CreateBitCast(Vec, InTy); 1774651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1775651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // extract_high is a shuffle on the second half of the input indices: E.g. 4, 1776651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // 5, 6, 7 if we're extracting <4 x i16> from <8 x i16>. 1777651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines for (unsigned i = 0; i < NumElts; ++i) 1778651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Indices.push_back(ConstantInt::get(Int32Ty, NumElts + i)); 1779651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1780651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Constant *Mask = ConstantVector::get(Indices); 1781651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateShuffleVector(VecCast, VecCast, Mask, "concat"); 1782651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 1783651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 178406b6c589a5fff8e5476fe2b4cd6a660af71bfdddBob Wilson/// GetPointeeAlignment - Given an expression with a pointer type, find the 178506b6c589a5fff8e5476fe2b4cd6a660af71bfdddBob Wilson/// alignment of the type referenced by the pointer. Skip over implicit 178606b6c589a5fff8e5476fe2b4cd6a660af71bfdddBob Wilson/// casts. 1787ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedmanstd::pair<llvm::Value*, unsigned> 1788ea93e40785ffeadfac66b948c95f9490ec26207aEli FriedmanCodeGenFunction::EmitPointerWithAlignment(const Expr *Addr) { 1789ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman assert(Addr->getType()->isPointerType()); 1790ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman Addr = Addr->IgnoreParens(); 1791ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman if (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(Addr)) { 1792a484fc73ec6331bcaad092270b4ab9c8d1df23c3Eli Friedman if ((ICE->getCastKind() == CK_BitCast || ICE->getCastKind() == CK_NoOp) && 1793a484fc73ec6331bcaad092270b4ab9c8d1df23c3Eli Friedman ICE->getSubExpr()->getType()->isPointerType()) { 1794258f930227c1a102c9c22eee88df65f748863425Jim Grosbach std::pair<llvm::Value*, unsigned> Ptr = 1795ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman EmitPointerWithAlignment(ICE->getSubExpr()); 1796ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman Ptr.first = Builder.CreateBitCast(Ptr.first, 1797ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman ConvertType(Addr->getType())); 1798ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman return Ptr; 1799ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman } else if (ICE->getCastKind() == CK_ArrayToPointerDecay) { 1800ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman LValue LV = EmitLValue(ICE->getSubExpr()); 18018e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman unsigned Align = LV.getAlignment().getQuantity(); 18028e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman if (!Align) { 18038e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman // FIXME: Once LValues are fixed to always set alignment, 18048e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman // zap this code. 18058e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman QualType PtTy = ICE->getSubExpr()->getType(); 18068e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman if (!PtTy->isIncompleteType()) 18078e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman Align = getContext().getTypeAlignInChars(PtTy).getQuantity(); 18088e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman else 18098e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman Align = 1; 18108e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman } 18118e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman return std::make_pair(LV.getAddress(), Align); 1812d6e73569ccf09369f9bd2021d53c88e7bded06e3Chris Lattner } 1813ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman } 1814ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(Addr)) { 1815ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman if (UO->getOpcode() == UO_AddrOf) { 1816ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman LValue LV = EmitLValue(UO->getSubExpr()); 18178e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman unsigned Align = LV.getAlignment().getQuantity(); 18188e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman if (!Align) { 18198e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman // FIXME: Once LValues are fixed to always set alignment, 18208e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman // zap this code. 18218e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman QualType PtTy = UO->getSubExpr()->getType(); 18228e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman if (!PtTy->isIncompleteType()) 18238e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman Align = getContext().getTypeAlignInChars(PtTy).getQuantity(); 18248e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman else 18258e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman Align = 1; 18268e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman } 18278e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman return std::make_pair(LV.getAddress(), Align); 182806b6c589a5fff8e5476fe2b4cd6a660af71bfdddBob Wilson } 182906b6c589a5fff8e5476fe2b4cd6a660af71bfdddBob Wilson } 1830f4c3db175101cbf94dad59419c3ed7aed51bf606Jay Foad 1831ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman unsigned Align = 1; 1832ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman QualType PtTy = Addr->getType()->getPointeeType(); 1833ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman if (!PtTy->isIncompleteType()) 1834ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman Align = getContext().getTypeAlignInChars(PtTy).getQuantity(); 1835ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman 1836ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman return std::make_pair(EmitScalarExpr(Addr), Align); 183706b6c589a5fff8e5476fe2b4cd6a660af71bfdddBob Wilson} 183806b6c589a5fff8e5476fe2b4cd6a660af71bfdddBob Wilson 1839651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesenum { 1840651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AddRetType = (1 << 0), 1841651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Add1ArgType = (1 << 1), 1842651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Add2ArgTypes = (1 << 2), 1843651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1844651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VectorizeRetType = (1 << 3), 1845651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VectorizeArgTypes = (1 << 4), 1846651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1847651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines InventFloatType = (1 << 5), 1848651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines UnsignedAlts = (1 << 6), 1849651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1850651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Use64BitVectors = (1 << 7), 1851651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Use128BitVectors = (1 << 8), 1852651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1853651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Vectorize1ArgType = Add1ArgType | VectorizeArgTypes, 1854651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VectorRet = AddRetType | VectorizeRetType, 1855651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VectorRetGetArgs01 = 1856651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AddRetType | Add2ArgTypes | VectorizeRetType | VectorizeArgTypes, 1857651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines FpCmpzModifiers = 1858651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AddRetType | VectorizeRetType | Add1ArgType | InventFloatType 1859651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines}; 1860651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1861651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines struct NeonIntrinsicInfo { 1862651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned BuiltinID; 1863651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned LLVMIntrinsic; 1864651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned AltLLVMIntrinsic; 1865651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const char *NameHint; 1866651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned TypeModifier; 1867651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1868651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool operator<(unsigned RHSBuiltinID) const { 1869651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return BuiltinID < RHSBuiltinID; 1870651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 1871651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines}; 1872651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1873651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#define NEONMAP0(NameBase) \ 1874651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines { NEON::BI__builtin_neon_ ## NameBase, 0, 0, #NameBase, 0 } 1875651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1876651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \ 1877651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines { NEON:: BI__builtin_neon_ ## NameBase, \ 1878651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Intrinsic::LLVMIntrinsic, 0, #NameBase, TypeModifier } 1879651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1880651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \ 1881651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines { NEON:: BI__builtin_neon_ ## NameBase, \ 1882651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \ 1883651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines #NameBase, TypeModifier } 1884651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1885651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic NeonIntrinsicInfo ARMSIMDIntrinsicMap [] = { 1886651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vabd_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts), 1887651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vabdq_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts), 1888651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vabs_v, arm_neon_vabs, 0), 1889651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vabsq_v, arm_neon_vabs, 0), 1890651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vaddhn_v), 1891651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vaesdq_v, arm_neon_aesd, 0), 1892651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vaeseq_v, arm_neon_aese, 0), 1893651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vaesimcq_v, arm_neon_aesimc, 0), 1894651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vaesmcq_v, arm_neon_aesmc, 0), 1895651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vbsl_v, arm_neon_vbsl, AddRetType), 1896651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vbslq_v, arm_neon_vbsl, AddRetType), 1897651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcage_v, arm_neon_vacge, 0), 1898651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcageq_v, arm_neon_vacge, 0), 1899651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcagt_v, arm_neon_vacgt, 0), 1900651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcagtq_v, arm_neon_vacgt, 0), 1901651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcale_v, arm_neon_vacge, 0), 1902651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcaleq_v, arm_neon_vacge, 0), 1903651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcalt_v, arm_neon_vacgt, 0), 1904651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcaltq_v, arm_neon_vacgt, 0), 1905651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcls_v, arm_neon_vcls, Add1ArgType), 1906651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vclsq_v, arm_neon_vcls, Add1ArgType), 1907651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vclz_v, ctlz, Add1ArgType), 1908651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vclzq_v, ctlz, Add1ArgType), 1909651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcnt_v, ctpop, Add1ArgType), 1910651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcntq_v, ctpop, Add1ArgType), 1911651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvt_f16_v, arm_neon_vcvtfp2hf, 0), 1912651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0), 1913651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvt_f32_v), 1914651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0), 1915651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0), 1916651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0), 1917651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0), 1918651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0), 1919651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvt_s32_v), 1920651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvt_s64_v), 1921651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvt_u32_v), 1922651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvt_u64_v), 1923651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0), 1924651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0), 1925651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0), 1926651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0), 1927651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0), 1928651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0), 1929651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0), 1930651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0), 1931651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0), 1932651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0), 1933651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0), 1934651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0), 1935651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0), 1936651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0), 1937651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0), 1938651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0), 1939651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0), 1940651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0), 1941651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0), 1942651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0), 1943651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0), 1944651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0), 1945651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0), 1946651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0), 1947651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0), 1948651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0), 1949651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0), 1950651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0), 1951651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0), 1952651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0), 1953651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0), 1954651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0), 1955651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvtq_f32_v), 1956651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0), 1957651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0), 1958651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0), 1959651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0), 1960651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0), 1961651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvtq_s32_v), 1962651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvtq_s64_v), 1963651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvtq_u32_v), 1964651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvtq_u64_v), 1965651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vext_v), 1966651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vextq_v), 1967651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vfma_v), 1968651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vfmaq_v), 1969651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vhadd_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts), 1970651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vhaddq_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts), 1971651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vhsub_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts), 1972651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vhsubq_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts), 1973651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vld1_dup_v), 1974651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld1_v, arm_neon_vld1, 0), 1975651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vld1q_dup_v), 1976651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld1q_v, arm_neon_vld1, 0), 1977651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0), 1978651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld2_v, arm_neon_vld2, 0), 1979651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0), 1980651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld2q_v, arm_neon_vld2, 0), 1981651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0), 1982651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld3_v, arm_neon_vld3, 0), 1983651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0), 1984651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld3q_v, arm_neon_vld3, 0), 1985651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0), 1986651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld4_v, arm_neon_vld4, 0), 1987651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0), 1988651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld4q_v, arm_neon_vld4, 0), 1989651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vmax_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts), 1990651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vmaxq_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts), 1991651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vmin_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts), 1992651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vminq_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts), 1993651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vmovl_v), 1994651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vmovn_v), 1995651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vmul_v, arm_neon_vmulp, Add1ArgType), 1996651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vmull_v), 1997651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vmulq_v, arm_neon_vmulp, Add1ArgType), 1998651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vpadal_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts), 1999651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vpadalq_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts), 2000651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vpadd_v, arm_neon_vpadd, Add1ArgType), 2001651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vpaddl_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts), 2002651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vpaddlq_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts), 2003651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vpaddq_v, arm_neon_vpadd, Add1ArgType), 2004651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vpmax_v, arm_neon_vpmaxu, arm_neon_vpmaxs, Add1ArgType | UnsignedAlts), 2005651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vpmin_v, arm_neon_vpminu, arm_neon_vpmins, Add1ArgType | UnsignedAlts), 2006651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqabs_v, arm_neon_vqabs, Add1ArgType), 2007651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqabsq_v, arm_neon_vqabs, Add1ArgType), 2008651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqadd_v, arm_neon_vqaddu, arm_neon_vqadds, Add1ArgType | UnsignedAlts), 2009651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqaddq_v, arm_neon_vqaddu, arm_neon_vqadds, Add1ArgType | UnsignedAlts), 2010651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqdmlal_v, arm_neon_vqdmull, arm_neon_vqadds, 0), 2011651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, arm_neon_vqsubs, 0), 2012651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqdmulh_v, arm_neon_vqdmulh, Add1ArgType), 2013651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqdmulhq_v, arm_neon_vqdmulh, Add1ArgType), 2014651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqdmull_v, arm_neon_vqdmull, Add1ArgType), 2015651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqmovn_v, arm_neon_vqmovnu, arm_neon_vqmovns, Add1ArgType | UnsignedAlts), 2016651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqmovun_v, arm_neon_vqmovnsu, Add1ArgType), 2017651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqneg_v, arm_neon_vqneg, Add1ArgType), 2018651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqnegq_v, arm_neon_vqneg, Add1ArgType), 2019651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqrdmulh_v, arm_neon_vqrdmulh, Add1ArgType), 2020651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqrdmulhq_v, arm_neon_vqrdmulh, Add1ArgType), 2021651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqrshl_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts), 2022651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqrshlq_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts), 2023651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqshl_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts), 2024651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqshl_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts), 2025651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqshlq_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts), 2026651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqshlq_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts), 2027651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqsub_v, arm_neon_vqsubu, arm_neon_vqsubs, Add1ArgType | UnsignedAlts), 2028651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqsubq_v, arm_neon_vqsubu, arm_neon_vqsubs, Add1ArgType | UnsignedAlts), 2029651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vraddhn_v, arm_neon_vraddhn, Add1ArgType), 2030651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0), 2031651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0), 2032651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vrecps_v, arm_neon_vrecps, Add1ArgType), 2033651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vrecpsq_v, arm_neon_vrecps, Add1ArgType), 2034651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vrhadd_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts), 2035651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vrhaddq_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts), 2036651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vrshl_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts), 2037651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vrshlq_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts), 2038651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0), 2039651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0), 2040651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vrsqrts_v, arm_neon_vrsqrts, Add1ArgType), 2041651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vrsqrtsq_v, arm_neon_vrsqrts, Add1ArgType), 2042651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vrsubhn_v, arm_neon_vrsubhn, Add1ArgType), 2043651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vsha1su0q_v, arm_neon_sha1su0, 0), 2044651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vsha1su1q_v, arm_neon_sha1su1, 0), 2045651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vsha256h2q_v, arm_neon_sha256h2, 0), 2046651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vsha256hq_v, arm_neon_sha256h, 0), 2047651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vsha256su0q_v, arm_neon_sha256su0, 0), 2048651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vsha256su1q_v, arm_neon_sha256su1, 0), 2049651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshl_n_v), 2050651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vshl_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts), 2051651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshll_n_v), 2052651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshlq_n_v), 2053651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vshlq_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts), 2054651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshr_n_v), 2055651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshrn_n_v), 2056651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshrq_n_v), 2057651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst1_v, arm_neon_vst1, 0), 2058651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst1q_v, arm_neon_vst1, 0), 2059651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0), 2060651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst2_v, arm_neon_vst2, 0), 2061651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0), 2062651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst2q_v, arm_neon_vst2, 0), 2063651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0), 2064651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst3_v, arm_neon_vst3, 0), 2065651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0), 2066651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst3q_v, arm_neon_vst3, 0), 2067651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0), 2068651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst4_v, arm_neon_vst4, 0), 2069651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0), 2070651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst4q_v, arm_neon_vst4, 0), 2071651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vsubhn_v), 2072651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vtrn_v), 2073651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vtrnq_v), 2074651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vtst_v), 2075651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vtstq_v), 2076651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vuzp_v), 2077651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vuzpq_v), 2078651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vzip_v), 2079651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vzipq_v) 2080651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines}; 2081651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 20826bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hinesstatic NeonIntrinsicInfo AArch64SIMDIntrinsicMap[] = { 20836bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vabs_v, aarch64_neon_abs, 0), 20846bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vabsq_v, aarch64_neon_abs, 0), 2085651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vaddhn_v), 20866bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaesdq_v, aarch64_crypto_aesd, 0), 20876bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaeseq_v, aarch64_crypto_aese, 0), 20886bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaesimcq_v, aarch64_crypto_aesimc, 0), 20896bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaesmcq_v, aarch64_crypto_aesmc, 0), 20906bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcage_v, aarch64_neon_facge, 0), 20916bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcageq_v, aarch64_neon_facge, 0), 20926bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcagt_v, aarch64_neon_facgt, 0), 20936bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0), 20946bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcale_v, aarch64_neon_facge, 0), 20956bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcaleq_v, aarch64_neon_facge, 0), 20966bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcalt_v, aarch64_neon_facgt, 0), 20976bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0), 20986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcls_v, aarch64_neon_cls, Add1ArgType), 20996bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vclsq_v, aarch64_neon_cls, Add1ArgType), 2100651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vclz_v, ctlz, Add1ArgType), 2101651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vclzq_v, ctlz, Add1ArgType), 2102651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcnt_v, ctpop, Add1ArgType), 2103651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcntq_v, ctpop, Add1ArgType), 21046bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvt_f16_v, aarch64_neon_vcvtfp2hf, 0), 21056bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0), 2106651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvt_f32_v), 21076bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 21086bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 21096bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0), 21106bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0), 21116bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0), 21126bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0), 2113651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvtq_f32_v), 21146bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 21156bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 21166bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0), 21176bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0), 21186bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0), 21196bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0), 21206bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtx_f32_v, aarch64_neon_fcvtxn, AddRetType | Add1ArgType), 2121651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vext_v), 2122651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vextq_v), 2123651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vfma_v), 2124651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vfmaq_v), 21256bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vhadd_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts), 21266bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vhaddq_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts), 21276bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vhsub_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts), 21286bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vhsubq_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts), 2129651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vmovl_v), 2130651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vmovn_v), 21316bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmul_v, aarch64_neon_pmul, Add1ArgType), 21326bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmulq_v, aarch64_neon_pmul, Add1ArgType), 21336bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpadd_v, aarch64_neon_addp, Add1ArgType), 21346bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vpaddl_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts), 21356bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vpaddlq_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts), 21366bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpaddq_v, aarch64_neon_addp, Add1ArgType), 21376bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqabs_v, aarch64_neon_sqabs, Add1ArgType), 21386bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqabsq_v, aarch64_neon_sqabs, Add1ArgType), 21396bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqadd_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts), 21406bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqaddq_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts), 21416bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0), 21426bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0), 21436bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqdmulh_v, aarch64_neon_sqdmulh, Add1ArgType), 21446bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqdmulhq_v, aarch64_neon_sqdmulh, Add1ArgType), 21456bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqdmull_v, aarch64_neon_sqdmull, Add1ArgType), 21466bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqmovn_v, aarch64_neon_uqxtn, aarch64_neon_sqxtn, Add1ArgType | UnsignedAlts), 21476bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovun_v, aarch64_neon_sqxtun, Add1ArgType), 21486bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqneg_v, aarch64_neon_sqneg, Add1ArgType), 21496bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqnegq_v, aarch64_neon_sqneg, Add1ArgType), 21506bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrdmulh_v, aarch64_neon_sqrdmulh, Add1ArgType), 21516bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrdmulhq_v, aarch64_neon_sqrdmulh, Add1ArgType), 21526bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqrshl_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts), 21536bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqrshlq_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts), 21546bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqshl_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl, UnsignedAlts), 21556bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqshl_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts), 21566bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqshlq_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl,UnsignedAlts), 21576bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqshlq_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts), 21586bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqsub_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts), 21596bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqsubq_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts), 21606bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vraddhn_v, aarch64_neon_raddhn, Add1ArgType), 21616bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0), 21626bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0), 21636bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrecps_v, aarch64_neon_frecps, Add1ArgType), 21646bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrecpsq_v, aarch64_neon_frecps, Add1ArgType), 21656bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vrhadd_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts), 21666bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vrhaddq_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts), 21676bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vrshl_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts), 21686bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vrshlq_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts), 21696bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0), 21706bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0), 21716bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrsqrts_v, aarch64_neon_frsqrts, Add1ArgType), 21726bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrsqrtsq_v, aarch64_neon_frsqrts, Add1ArgType), 21736bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrsubhn_v, aarch64_neon_rsubhn, Add1ArgType), 21746bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha1su0q_v, aarch64_crypto_sha1su0, 0), 21756bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha1su1q_v, aarch64_crypto_sha1su1, 0), 21766bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha256h2q_v, aarch64_crypto_sha256h2, 0), 21776bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha256hq_v, aarch64_crypto_sha256h, 0), 21786bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha256su0q_v, aarch64_crypto_sha256su0, 0), 21796bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha256su1q_v, aarch64_crypto_sha256su1, 0), 2180651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshl_n_v), 21816bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vshl_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts), 2182651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshll_n_v), 2183651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshlq_n_v), 21846bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vshlq_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts), 2185651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshr_n_v), 2186651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshrn_n_v), 2187651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshrq_n_v), 2188651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vsubhn_v), 2189651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vtst_v), 2190651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vtstq_v), 2191651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines}; 2192651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 21936bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hinesstatic NeonIntrinsicInfo AArch64SISDIntrinsicMap[] = { 21946bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vabdd_f64, aarch64_sisd_fabd, Add1ArgType), 21956bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vabds_f32, aarch64_sisd_fabd, Add1ArgType), 21966bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vabsd_s64, aarch64_neon_abs, Add1ArgType), 21976bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddlv_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType), 21986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddlv_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType), 21996bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddlvq_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType), 22006bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddlvq_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType), 22016bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddv_f32, aarch64_neon_faddv, AddRetType | Add1ArgType), 22026bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddv_s32, aarch64_neon_saddv, AddRetType | Add1ArgType), 22036bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddv_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType), 22046bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddvq_f32, aarch64_neon_faddv, AddRetType | Add1ArgType), 22056bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddvq_f64, aarch64_neon_faddv, AddRetType | Add1ArgType), 22066bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddvq_s32, aarch64_neon_saddv, AddRetType | Add1ArgType), 22076bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddvq_s64, aarch64_neon_saddv, AddRetType | Add1ArgType), 22086bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddvq_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType), 22096bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddvq_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType), 22106bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcaged_f64, aarch64_neon_facge, AddRetType | Add1ArgType), 22116bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcages_f32, aarch64_neon_facge, AddRetType | Add1ArgType), 22126bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcagtd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType), 22136bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcagts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType), 22146bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcaled_f64, aarch64_neon_facge, AddRetType | Add1ArgType), 22156bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcales_f32, aarch64_neon_facge, AddRetType | Add1ArgType), 22166bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcaltd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType), 22176bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcalts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType), 22186bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtad_s64_f64, aarch64_neon_fcvtas, AddRetType | Add1ArgType), 22196bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtad_u64_f64, aarch64_neon_fcvtau, AddRetType | Add1ArgType), 22206bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtas_s32_f32, aarch64_neon_fcvtas, AddRetType | Add1ArgType), 22216bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtas_u32_f32, aarch64_neon_fcvtau, AddRetType | Add1ArgType), 22226bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtd_n_f64_s64, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType), 22236bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtd_n_f64_u64, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType), 22246bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtd_n_s64_f64, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType), 22256bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtd_n_u64_f64, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType), 22266bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtmd_s64_f64, aarch64_neon_fcvtms, AddRetType | Add1ArgType), 22276bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtmd_u64_f64, aarch64_neon_fcvtmu, AddRetType | Add1ArgType), 22286bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtms_s32_f32, aarch64_neon_fcvtms, AddRetType | Add1ArgType), 22296bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtms_u32_f32, aarch64_neon_fcvtmu, AddRetType | Add1ArgType), 22306bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtnd_s64_f64, aarch64_neon_fcvtns, AddRetType | Add1ArgType), 22316bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtnd_u64_f64, aarch64_neon_fcvtnu, AddRetType | Add1ArgType), 22326bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtns_s32_f32, aarch64_neon_fcvtns, AddRetType | Add1ArgType), 22336bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtns_u32_f32, aarch64_neon_fcvtnu, AddRetType | Add1ArgType), 22346bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtpd_s64_f64, aarch64_neon_fcvtps, AddRetType | Add1ArgType), 22356bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtpd_u64_f64, aarch64_neon_fcvtpu, AddRetType | Add1ArgType), 22366bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtps_s32_f32, aarch64_neon_fcvtps, AddRetType | Add1ArgType), 22376bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtps_u32_f32, aarch64_neon_fcvtpu, AddRetType | Add1ArgType), 22386bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvts_n_f32_s32, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType), 22396bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvts_n_f32_u32, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType), 22406bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvts_n_s32_f32, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType), 22416bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvts_n_u32_f32, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType), 22426bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0), 22436bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxnmv_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 22446bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxnmvq_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 22456bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxnmvq_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 22466bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxv_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 22476bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxv_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType), 22486bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxv_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType), 22496bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxvq_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 22506bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxvq_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 22516bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxvq_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType), 22526bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxvq_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType), 22536bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminnmv_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 22546bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminnmvq_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 22556bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminnmvq_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 22566bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminv_f32, aarch64_neon_fminv, AddRetType | Add1ArgType), 22576bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminv_s32, aarch64_neon_sminv, AddRetType | Add1ArgType), 22586bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminv_u32, aarch64_neon_uminv, AddRetType | Add1ArgType), 22596bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminvq_f32, aarch64_neon_fminv, AddRetType | Add1ArgType), 22606bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminvq_f64, aarch64_neon_fminv, AddRetType | Add1ArgType), 22616bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminvq_s32, aarch64_neon_sminv, AddRetType | Add1ArgType), 22626bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminvq_u32, aarch64_neon_uminv, AddRetType | Add1ArgType), 22636bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0), 22646bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmulxd_f64, aarch64_neon_fmulx, Add1ArgType), 22656bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmulxs_f32, aarch64_neon_fmulx, Add1ArgType), 22666bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpaddd_s64, aarch64_neon_uaddv, AddRetType | Add1ArgType), 22676bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpaddd_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType), 22686bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpmaxnmqd_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 22696bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpmaxnms_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 22706bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpmaxqd_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 22716bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpmaxs_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 22726bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpminnmqd_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 22736bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpminnms_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 22746bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpminqd_f64, aarch64_neon_fminv, AddRetType | Add1ArgType), 22756bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpmins_f32, aarch64_neon_fminv, AddRetType | Add1ArgType), 22766bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqabsb_s8, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors), 22776bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqabsd_s64, aarch64_neon_sqabs, Add1ArgType), 22786bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqabsh_s16, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors), 22796bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqabss_s32, aarch64_neon_sqabs, Add1ArgType), 22806bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqaddb_s8, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors), 22816bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqaddb_u8, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors), 22826bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqaddd_s64, aarch64_neon_sqadd, Add1ArgType), 22836bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqaddd_u64, aarch64_neon_uqadd, Add1ArgType), 22846bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqaddh_s16, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors), 22856bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqaddh_u16, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors), 22866bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqadds_s32, aarch64_neon_sqadd, Add1ArgType), 22876bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqadds_u32, aarch64_neon_uqadd, Add1ArgType), 22886bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqdmulhh_s16, aarch64_neon_sqdmulh, Vectorize1ArgType | Use64BitVectors), 22896bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqdmulhs_s32, aarch64_neon_sqdmulh, Add1ArgType), 22906bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqdmullh_s16, aarch64_neon_sqdmull, VectorRet | Use128BitVectors), 22916bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0), 22926bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovnd_s64, aarch64_neon_scalar_sqxtn, AddRetType | Add1ArgType), 22936bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovnd_u64, aarch64_neon_scalar_uqxtn, AddRetType | Add1ArgType), 22946bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovnh_s16, aarch64_neon_sqxtn, VectorRet | Use64BitVectors), 22956bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovnh_u16, aarch64_neon_uqxtn, VectorRet | Use64BitVectors), 22966bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovns_s32, aarch64_neon_sqxtn, VectorRet | Use64BitVectors), 22976bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovns_u32, aarch64_neon_uqxtn, VectorRet | Use64BitVectors), 22986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovund_s64, aarch64_neon_scalar_sqxtun, AddRetType | Add1ArgType), 22996bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovunh_s16, aarch64_neon_sqxtun, VectorRet | Use64BitVectors), 23006bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovuns_s32, aarch64_neon_sqxtun, VectorRet | Use64BitVectors), 23016bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqnegb_s8, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors), 23026bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqnegd_s64, aarch64_neon_sqneg, Add1ArgType), 23036bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqnegh_s16, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors), 23046bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqnegs_s32, aarch64_neon_sqneg, Add1ArgType), 23056bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrdmulhh_s16, aarch64_neon_sqrdmulh, Vectorize1ArgType | Use64BitVectors), 23066bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrdmulhs_s32, aarch64_neon_sqrdmulh, Add1ArgType), 23076bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshlb_s8, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors), 23086bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshlb_u8, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors), 23096bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshld_s64, aarch64_neon_sqrshl, Add1ArgType), 23106bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshld_u64, aarch64_neon_uqrshl, Add1ArgType), 23116bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshlh_s16, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors), 23126bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshlh_u16, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors), 23136bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshls_s32, aarch64_neon_sqrshl, Add1ArgType), 23146bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshls_u32, aarch64_neon_uqrshl, Add1ArgType), 23156bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshrnd_n_s64, aarch64_neon_sqrshrn, AddRetType), 23166bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshrnd_n_u64, aarch64_neon_uqrshrn, AddRetType), 23176bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshrnh_n_s16, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors), 23186bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshrnh_n_u16, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors), 23196bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshrns_n_s32, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors), 23206bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshrns_n_u32, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors), 23216bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshrund_n_s64, aarch64_neon_sqrshrun, AddRetType), 23226bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshrunh_n_s16, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors), 23236bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshruns_n_s32, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors), 23246bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlb_n_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 23256bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlb_n_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 23266bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlb_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 23276bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlb_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 23286bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshld_s64, aarch64_neon_sqshl, Add1ArgType), 23296bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshld_u64, aarch64_neon_uqshl, Add1ArgType), 23306bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlh_n_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 23316bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlh_n_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 23326bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlh_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 23336bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlh_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 23346bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshls_n_s32, aarch64_neon_sqshl, Add1ArgType), 23356bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshls_n_u32, aarch64_neon_uqshl, Add1ArgType), 23366bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshls_s32, aarch64_neon_sqshl, Add1ArgType), 23376bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshls_u32, aarch64_neon_uqshl, Add1ArgType), 23386bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlub_n_s8, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors), 23396bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshluh_n_s16, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors), 23406bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlus_n_s32, aarch64_neon_sqshlu, Add1ArgType), 23416bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshrnd_n_s64, aarch64_neon_sqshrn, AddRetType), 23426bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshrnd_n_u64, aarch64_neon_uqshrn, AddRetType), 23436bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshrnh_n_s16, aarch64_neon_sqshrn, VectorRet | Use64BitVectors), 23446bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshrnh_n_u16, aarch64_neon_uqshrn, VectorRet | Use64BitVectors), 23456bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshrns_n_s32, aarch64_neon_sqshrn, VectorRet | Use64BitVectors), 23466bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshrns_n_u32, aarch64_neon_uqshrn, VectorRet | Use64BitVectors), 23476bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshrund_n_s64, aarch64_neon_sqshrun, AddRetType), 23486bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshrunh_n_s16, aarch64_neon_sqshrun, VectorRet | Use64BitVectors), 23496bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshruns_n_s32, aarch64_neon_sqshrun, VectorRet | Use64BitVectors), 23506bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqsubb_s8, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors), 23516bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqsubb_u8, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors), 23526bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqsubd_s64, aarch64_neon_sqsub, Add1ArgType), 23536bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqsubd_u64, aarch64_neon_uqsub, Add1ArgType), 23546bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqsubh_s16, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors), 23556bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqsubh_u16, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors), 23566bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqsubs_s32, aarch64_neon_sqsub, Add1ArgType), 23576bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqsubs_u32, aarch64_neon_uqsub, Add1ArgType), 23586bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrecped_f64, aarch64_neon_frecpe, Add1ArgType), 23596bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrecpes_f32, aarch64_neon_frecpe, Add1ArgType), 23606bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrecpxd_f64, aarch64_neon_frecpx, Add1ArgType), 23616bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrecpxs_f32, aarch64_neon_frecpx, Add1ArgType), 23626bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrshld_s64, aarch64_neon_srshl, Add1ArgType), 23636bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrshld_u64, aarch64_neon_urshl, Add1ArgType), 23646bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrsqrted_f64, aarch64_neon_frsqrte, Add1ArgType), 23656bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrsqrtes_f32, aarch64_neon_frsqrte, Add1ArgType), 23666bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrsqrtsd_f64, aarch64_neon_frsqrts, Add1ArgType), 23676bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrsqrtss_f32, aarch64_neon_frsqrts, Add1ArgType), 23686bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0), 23696bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0), 23706bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0), 23716bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0), 23726bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vshld_s64, aarch64_neon_sshl, Add1ArgType), 23736bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vshld_u64, aarch64_neon_ushl, Add1ArgType), 23746bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vslid_n_s64, aarch64_neon_vsli, Vectorize1ArgType), 23756bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vslid_n_u64, aarch64_neon_vsli, Vectorize1ArgType), 23766bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsqaddb_u8, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors), 23776bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsqaddd_u64, aarch64_neon_usqadd, Add1ArgType), 23786bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsqaddh_u16, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors), 23796bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsqadds_u32, aarch64_neon_usqadd, Add1ArgType), 23806bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsrid_n_s64, aarch64_neon_vsri, Vectorize1ArgType), 23816bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsrid_n_u64, aarch64_neon_vsri, Vectorize1ArgType), 23826bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vuqaddb_s8, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors), 23836bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vuqaddd_s64, aarch64_neon_suqadd, Add1ArgType), 23846bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vuqaddh_s16, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors), 23856bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vuqadds_s32, aarch64_neon_suqadd, Add1ArgType), 2386651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines}; 2387651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2388651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#undef NEONMAP0 2389651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#undef NEONMAP1 2390651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#undef NEONMAP2 2391651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2392651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic bool NEONSIMDIntrinsicsProvenSorted = false; 2393651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 23946bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hinesstatic bool AArch64SIMDIntrinsicsProvenSorted = false; 23956bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hinesstatic bool AArch64SISDIntrinsicsProvenSorted = false; 2396651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2397651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2398651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic const NeonIntrinsicInfo * 2399651f13cea278ec967336033dd032faef0e9fc2ecStephen HinesfindNeonIntrinsicInMap(llvm::ArrayRef<NeonIntrinsicInfo> IntrinsicMap, 2400651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned BuiltinID, bool &MapProvenSorted) { 2401651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2402651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#ifndef NDEBUG 2403651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (!MapProvenSorted) { 2404651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: use std::is_sorted once C++11 is allowed 2405651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines for (unsigned i = 0; i < IntrinsicMap.size() - 1; ++i) 2406651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines assert(IntrinsicMap[i].BuiltinID <= IntrinsicMap[i + 1].BuiltinID); 2407651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines MapProvenSorted = true; 2408651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2409651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#endif 2410651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2411651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const NeonIntrinsicInfo *Builtin = 2412651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines std::lower_bound(IntrinsicMap.begin(), IntrinsicMap.end(), BuiltinID); 2413651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2414651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID) 2415651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builtin; 2416651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 24176bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 2418651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 2419651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2420651f13cea278ec967336033dd032faef0e9fc2ecStephen HinesFunction *CodeGenFunction::LookupNeonLLVMIntrinsic(unsigned IntrinsicID, 2421651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Modifier, 2422651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *ArgType, 2423651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const CallExpr *E) { 2424651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines int VectorSize = 0; 2425651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Modifier & Use64BitVectors) 2426651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VectorSize = 64; 2427651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines else if (Modifier & Use128BitVectors) 2428651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VectorSize = 128; 2429651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2430651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Return type. 2431651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<llvm::Type *, 3> Tys; 2432651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Modifier & AddRetType) { 2433651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty = ConvertType(E->getCallReturnType()); 2434651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Modifier & VectorizeRetType) 2435651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::VectorType::get( 2436651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1); 2437651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2438651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Tys.push_back(Ty); 2439651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2440651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2441651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Arguments. 2442651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Modifier & VectorizeArgTypes) { 2443651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1; 2444651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ArgType = llvm::VectorType::get(ArgType, Elts); 2445651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2446651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2447651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Modifier & (Add1ArgType | Add2ArgTypes)) 2448651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Tys.push_back(ArgType); 2449651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2450651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Modifier & Add2ArgTypes) 2451651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Tys.push_back(ArgType); 2452651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2453651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Modifier & InventFloatType) 2454651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Tys.push_back(FloatTy); 2455651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2456651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return CGM.getIntrinsic(IntrinsicID, Tys); 2457651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 2458651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2459651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic Value *EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, 2460651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const NeonIntrinsicInfo &SISDInfo, 2461651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVectorImpl<Value *> &Ops, 2462651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const CallExpr *E) { 2463651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned BuiltinID = SISDInfo.BuiltinID; 2464651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned int Int = SISDInfo.LLVMIntrinsic; 2465651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Modifier = SISDInfo.TypeModifier; 2466651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const char *s = SISDInfo.NameHint; 2467651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2468651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 2469651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_s64: 2470651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_u64: 2471651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcles_f32: 2472651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_f64: 2473651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_s64: 2474651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_u64: 2475651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclts_f32: 2476651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_f64: 2477651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcales_f32: 2478651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcaled_f64: 2479651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcalts_f32: 2480651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcaltd_f64: 2481651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Only one direction of comparisons actually exist, cmle is actually a cmge 2482651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // with swapped operands. The table gives us the right intrinsic but we 2483651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // still need to do the swap. 2484651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines std::swap(Ops[0], Ops[1]); 2485651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 2486651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2487651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2488651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines assert(Int && "Generic code assumes a valid intrinsic"); 2489651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2490651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Determine the type(s) of this overloaded AArch64 intrinsic. 2491651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const Expr *Arg = E->getArg(0); 2492651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *ArgTy = CGF.ConvertType(Arg->getType()); 2493651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGF.LookupNeonLLVMIntrinsic(Int, Modifier, ArgTy, E); 2494651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2495651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines int j = 0; 2496651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ConstantInt *C0 = ConstantInt::get(CGF.Int32Ty, 0); 2497651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end(); 2498651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ai != ae; ++ai, ++j) { 2499651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *ArgTy = ai->getType(); 2500651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Ops[j]->getType()->getPrimitiveSizeInBits() == 2501651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ArgTy->getPrimitiveSizeInBits()) 2502651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines continue; 2503651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2504651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy()); 2505651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The constant argument to an _n_ intrinsic always has Int32Ty, so truncate 2506651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // it before inserting. 2507651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[j] = 2508651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.CreateTruncOrBitCast(Ops[j], ArgTy->getVectorElementType()); 2509651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[j] = 2510651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.CreateInsertElement(UndefValue::get(ArgTy), Ops[j], C0); 2511651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2512651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2513651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Result = CGF.EmitNeonCall(F, Ops, s); 2514651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *ResultType = CGF.ConvertType(E->getType()); 2515651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (ResultType->getPrimitiveSizeInBits() < 2516651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Result->getType()->getPrimitiveSizeInBits()) 2517651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return CGF.Builder.CreateExtractElement(Result, C0); 2518651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2519651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return CGF.Builder.CreateBitCast(Result, ResultType, s); 2520651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 2521651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2522651f13cea278ec967336033dd032faef0e9fc2ecStephen HinesValue *CodeGenFunction::EmitCommonNeonBuiltinExpr( 2523651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic, 2524651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const char *NameHint, unsigned Modifier, const CallExpr *E, 2525651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVectorImpl<llvm::Value *> &Ops, llvm::Value *Align) { 25262e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu // Get the last argument, which specifies the vector type. 2527651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::APSInt NeonTypeConst; 25282e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu const Expr *Arg = E->getArg(E->getNumArgs() - 1); 2529651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (!Arg->isIntegerConstantExpr(NeonTypeConst, getContext())) 25306bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 25312e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu 25322e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu // Determine the type of this overloaded NEON intrinsic. 2533651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags Type(NeonTypeConst.getZExtValue()); 2534651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Usgn = Type.isUnsigned(); 2535651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Quad = Type.isQuad(); 2536651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2537651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType *VTy = GetNeonType(this, Type); 25382e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu llvm::Type *Ty = VTy; 25392e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu if (!Ty) 25406bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 25412e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu 2542651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Int = LLVMIntrinsic; 2543651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if ((Modifier & UnsignedAlts) && !Usgn) 2544651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = AltLLVMIntrinsic; 25452e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu 25462e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu switch (BuiltinID) { 2547651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines default: break; 2548651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vabs_v: 2549651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vabsq_v: 2550651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (VTy->getElementType()->isFloatingPointTy()) 2551651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs"); 2552651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vabs"); 2553651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddhn_v: { 2554651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType *SrcTy = 2555651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::getExtendedElementVectorType(VTy); 2556651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2557651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // %sum = add <4 x i32> %lhs, %rhs 2558651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 2559651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); 2560651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn"); 2561651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2562651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16> 2563651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Constant *ShiftAmt = ConstantInt::get(SrcTy->getElementType(), 2564651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SrcTy->getScalarSizeInBits() / 2); 2565651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ShiftAmt = ConstantVector::getSplat(VTy->getNumElements(), ShiftAmt); 2566651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn"); 2567651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2568651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // %res = trunc <4 x i32> %high to <4 x i16> 2569651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], VTy, "vaddhn"); 25702e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu } 2571651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcale_v: 2572651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcaleq_v: 2573651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcalt_v: 2574651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcaltq_v: 2575651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines std::swap(Ops[0], Ops[1]); 2576651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcage_v: 2577651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcageq_v: 2578651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcagt_v: 2579651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcagtq_v: { 2580651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *VecFlt = llvm::VectorType::get( 2581651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy->getScalarSizeInBits() == 32 ? FloatTy : DoubleTy, 2582651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy->getNumElements()); 2583651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[] = { VTy, VecFlt }; 2584651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); 2585651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(F, Ops, NameHint); 2586651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2587651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclz_v: 2588651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclzq_v: 2589651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // We generate target-independent intrinsic, which needs a second argument 2590651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // for whether or not clz of zero is undefined; on ARM it isn't. 2591651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef())); 2592651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 2593651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_f32_v: 2594651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_f32_v: 2595651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 2596651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, Quad)); 2597651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") 2598651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); 2599651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_n_f32_v: 2600651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_n_f64_v: 2601651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_n_f32_v: 2602651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_n_f64_v: { 2603651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Double = 2604651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 2605651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *FloatTy = 2606651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines GetNeonType(this, NeonTypeFlags(Double ? NeonTypeFlags::Float64 2607651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : NeonTypeFlags::Float32, 2608651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines false, Quad)); 2609651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { FloatTy, Ty }; 2610651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic; 2611651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(Int, Tys); 2612651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(F, Ops, "vcvt_n"); 26132e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu } 2614651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_n_s32_v: 2615651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_n_u32_v: 2616651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_n_s64_v: 2617651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_n_u64_v: 2618651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_n_s32_v: 2619651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_n_u32_v: 2620651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_n_s64_v: 2621651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_n_u64_v: { 2622651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Double = 2623651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 2624651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *FloatTy = 2625651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines GetNeonType(this, NeonTypeFlags(Double ? NeonTypeFlags::Float64 2626651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : NeonTypeFlags::Float32, 2627651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines false, Quad)); 2628651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, FloatTy }; 2629651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); 2630651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(F, Ops, "vcvt_n"); 26312e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu } 2632651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_s32_v: 2633651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_u32_v: 2634651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_s64_v: 2635651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_u64_v: 2636651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_s32_v: 2637651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_u32_v: 2638651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_s64_v: 2639651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_u64_v: { 2640651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Double = 2641651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 2642651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *FloatTy = 2643651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines GetNeonType(this, NeonTypeFlags(Double ? NeonTypeFlags::Float64 2644651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : NeonTypeFlags::Float32, 2645651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines false, Quad)); 2646651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy); 2647651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt") 2648651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : Builder.CreateFPToSI(Ops[0], Ty, "vcvt"); 26492e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu } 2650651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvta_s32_v: 2651651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvta_s64_v: 2652651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvta_u32_v: 2653651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvta_u64_v: 2654651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtaq_s32_v: 2655651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtaq_s64_v: 2656651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtaq_u32_v: 2657651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtaq_u64_v: 2658651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtn_s32_v: 2659651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtn_s64_v: 2660651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtn_u32_v: 2661651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtn_u64_v: 2662651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtnq_s32_v: 2663651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtnq_s64_v: 2664651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtnq_u32_v: 2665651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtnq_u64_v: 2666651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtp_s32_v: 2667651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtp_s64_v: 2668651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtp_u32_v: 2669651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtp_u64_v: 2670651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtpq_s32_v: 2671651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtpq_s64_v: 2672651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtpq_u32_v: 2673651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtpq_u64_v: 2674651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtm_s32_v: 2675651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtm_s64_v: 2676651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtm_u32_v: 2677651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtm_u64_v: 2678651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtmq_s32_v: 2679651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtmq_s64_v: 2680651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtmq_u32_v: 2681651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtmq_u64_v: { 2682651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Double = 2683651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 2684651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *InTy = 2685651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines GetNeonType(this, 2686651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags(Double ? NeonTypeFlags::Float64 2687651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : NeonTypeFlags::Float32, false, Quad)); 2688651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, InTy }; 2689651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint); 2690651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2691651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vext_v: 2692651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vextq_v: { 2693651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines int CV = cast<ConstantInt>(Ops[2])->getSExtValue(); 2694651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Constant*, 16> Indices; 2695651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) 2696651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Indices.push_back(ConstantInt::get(Int32Ty, i+CV)); 26972e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu 2698651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 2699651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 2700651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *SV = llvm::ConstantVector::get(Indices); 2701651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateShuffleVector(Ops[0], Ops[1], SV, "vext"); 27022e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu } 2703651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfma_v: 2704651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfmaq_v: { 2705651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 2706651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 2707651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 2708651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 27092e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu 2710651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // NEON intrinsic puts accumulator first, unlike the LLVM fma. 2711651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]); 2712651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2713651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_v: 2714651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_v: 2715651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Align); 2716651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vld1"); 2717651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_v: 2718651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2q_v: 2719651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_v: 2720651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3q_v: 2721651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_v: 2722651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4q_v: { 2723651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(LLVMIntrinsic, Ty); 2724651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall2(F, Ops[1], Align, NameHint); 2725651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 2726651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 2727651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateStore(Ops[1], Ops[0]); 2728651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2729651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_dup_v: 2730651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_dup_v: { 2731651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *V = UndefValue::get(Ty); 2732651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::PointerType::getUnqual(VTy->getElementType()); 2733651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 2734651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines LoadInst *Ld = Builder.CreateLoad(Ops[0]); 2735651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ld->setAlignment(cast<ConstantInt>(Align)->getZExtValue()); 2736651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Constant *CI = ConstantInt::get(Int32Ty, 0); 2737651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateInsertElement(V, Ld, CI); 2738651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonSplat(Ops[0], CI); 2739651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2740651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_lane_v: 2741651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2q_lane_v: 2742651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_lane_v: 2743651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3q_lane_v: 2744651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_lane_v: 2745651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4q_lane_v: { 2746651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(LLVMIntrinsic, Ty); 2747651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines for (unsigned I = 2; I < Ops.size() - 1; ++I) 2748651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[I] = Builder.CreateBitCast(Ops[I], Ty); 2749651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Align); 2750651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), NameHint); 2751651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 2752651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 2753651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateStore(Ops[1], Ops[0]); 2754651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2755651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmovl_v: { 2756651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy); 2757651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], DTy); 2758651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Usgn) 2759651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateZExt(Ops[0], Ty, "vmovl"); 2760651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateSExt(Ops[0], Ty, "vmovl"); 2761651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2762651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmovn_v: { 2763651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *QTy = llvm::VectorType::getExtendedElementVectorType(VTy); 2764651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], QTy); 2765651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], Ty, "vmovn"); 2766651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2767651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmull_v: 2768651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: the integer vmull operations could be emitted in terms of pure 2769651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // LLVM IR (2 exts followed by a mul). Unfortunately LLVM has a habit of 2770651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // hoisting the exts outside loops. Until global ISel comes along that can 2771651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // see through such movement this leads to bad CodeGen. So we need an 2772651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // intrinsic for now. 2773651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls; 2774651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp : Int; 2775651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); 2776651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpadal_v: 2777651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpadalq_v: { 2778651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The source operand type has twice as many elements of half the size. 2779651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits(); 2780651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *EltTy = 2781651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), EltBits / 2); 2782651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *NarrowTy = 2783651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(EltTy, VTy->getNumElements() * 2); 2784651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, NarrowTy }; 2785651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint); 2786651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2787651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpaddl_v: 2788651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpaddlq_v: { 2789651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The source operand type has twice as many elements of half the size. 2790651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits(); 2791651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *EltTy = llvm::IntegerType::get(getLLVMContext(), EltBits / 2); 2792651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *NarrowTy = 2793651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(EltTy, VTy->getNumElements() * 2); 2794651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, NarrowTy }; 2795651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl"); 2796651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2797651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlal_v: 2798651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlsl_v: { 2799651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end()); 2800651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Mul = EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), 2801651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines MulOps, "vqdmlal"); 2802651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2803651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value *, 2> AccumOps; 2804651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AccumOps.push_back(Ops[0]); 2805651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AccumOps.push_back(Mul); 2806651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), 2807651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AccumOps, NameHint); 2808651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2809651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshl_n_v: 2810651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshlq_n_v: 2811651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n", 2812651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1, false); 2813651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrecpe_v: 2814651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrecpeq_v: 2815651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrsqrte_v: 2816651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrsqrteq_v: 2817651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic; 2818651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint); 2819651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2820651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshl_n_v: 2821651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshlq_n_v: 2822651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false); 2823651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1], 2824651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vshl_n"); 2825651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshll_n_v: { 2826651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *SrcTy = llvm::VectorType::getTruncatedElementVectorType(VTy); 2827651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 2828651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Usgn) 2829651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateZExt(Ops[0], VTy); 2830651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines else 2831651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateSExt(Ops[0], VTy); 2832651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = EmitNeonShiftVector(Ops[1], VTy, false); 2833651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateShl(Ops[0], Ops[1], "vshll_n"); 2834651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2835651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshrn_n_v: { 2836651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *SrcTy = llvm::VectorType::getExtendedElementVectorType(VTy); 2837651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 2838651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy, false); 2839651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Usgn) 2840651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]); 2841651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines else 2842651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]); 2843651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n"); 2844651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2845651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshr_n_v: 2846651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshrq_n_v: 2847651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn, "vshr_n"); 2848651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_v: 2849651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_v: 2850651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2_v: 2851651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2q_v: 2852651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3_v: 2853651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3q_v: 2854651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4_v: 2855651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4q_v: 2856651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2_lane_v: 2857651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2q_lane_v: 2858651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3_lane_v: 2859651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3q_lane_v: 2860651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4_lane_v: 2861651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4q_lane_v: 2862651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Align); 2863651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, ""); 2864651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsubhn_v: { 2865651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType *SrcTy = 2866651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::getExtendedElementVectorType(VTy); 2867651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2868651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // %sum = add <4 x i32> %lhs, %rhs 2869651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 2870651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); 2871651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn"); 2872651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2873651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16> 2874651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Constant *ShiftAmt = ConstantInt::get(SrcTy->getElementType(), 2875651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SrcTy->getScalarSizeInBits() / 2); 2876651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ShiftAmt = ConstantVector::getSplat(VTy->getNumElements(), ShiftAmt); 2877651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn"); 2878651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2879651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // %res = trunc <4 x i32> %high to <4 x i16> 2880651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], VTy, "vsubhn"); 2881651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2882651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtrn_v: 2883651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtrnq_v: { 2884651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 2885651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 2886651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 28876bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *SV = nullptr; 2888651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2889651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines for (unsigned vi = 0; vi != 2; ++vi) { 2890651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Constant*, 16> Indices; 2891651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 2892651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Indices.push_back(Builder.getInt32(i+vi)); 2893651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Indices.push_back(Builder.getInt32(i+e+vi)); 2894651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2895651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); 28966bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SV = llvm::ConstantVector::get(Indices); 28976bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn"); 28986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SV = Builder.CreateStore(SV, Addr); 2899651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 29006bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return SV; 29016bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } 29026bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case NEON::BI__builtin_neon_vtst_v: 29036bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case NEON::BI__builtin_neon_vtstq_v: { 2904651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 2905651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 29066bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); 29076bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], 29086bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ConstantAggregateZero::get(Ty)); 29096bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return Builder.CreateSExt(Ops[0], Ty, "vtst"); 29106bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } 29116bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case NEON::BI__builtin_neon_vuzp_v: 29126bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case NEON::BI__builtin_neon_vuzpq_v: { 29136bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 29146bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 29156bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 29166bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *SV = nullptr; 2917651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 29186bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines for (unsigned vi = 0; vi != 2; ++vi) { 29196bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SmallVector<Constant*, 16> Indices; 29206bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) 29216bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Indices.push_back(ConstantInt::get(Int32Ty, 2*i+vi)); 2922651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 29236bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); 29246bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SV = llvm::ConstantVector::get(Indices); 29256bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp"); 29266bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SV = Builder.CreateStore(SV, Addr); 29276bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } 29286bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return SV; 2929651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 29306bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case NEON::BI__builtin_neon_vzip_v: 29316bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case NEON::BI__builtin_neon_vzipq_v: { 29326bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 2933651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 2934651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 29356bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *SV = nullptr; 2936651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 29376bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines for (unsigned vi = 0; vi != 2; ++vi) { 29386bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SmallVector<Constant*, 16> Indices; 29396bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 29406bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Indices.push_back(ConstantInt::get(Int32Ty, (i + vi*e) >> 1)); 29416bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Indices.push_back(ConstantInt::get(Int32Ty, ((i + vi*e) >> 1)+e)); 29426bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } 29436bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); 29446bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SV = llvm::ConstantVector::get(Indices); 29456bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip"); 29466bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SV = Builder.CreateStore(SV, Addr); 29476bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } 29486bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return SV; 2949651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2950651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 29516bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 29526bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines assert(Int && "Expected valid intrinsic number"); 29536bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 29546bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // Determine the type(s) of this overloaded AArch64 intrinsic. 29556bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = LookupNeonLLVMIntrinsic(Int, Modifier, Ty, E); 29566bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 29576bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *Result = EmitNeonCall(F, Ops, NameHint); 29586bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines llvm::Type *ResultType = ConvertType(E->getType()); 29596bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // AArch64 intrinsic one-element vector type cast to 29606bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // scalar type expected by the builtin 29616bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return Builder.CreateBitCast(Result, ResultType, NameHint); 29626bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines} 29636bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 29646bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen HinesValue *CodeGenFunction::EmitAArch64CompareBuiltinExpr( 29656bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *Op, llvm::Type *Ty, const CmpInst::Predicate Fp, 29666bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines const CmpInst::Predicate Ip, const Twine &Name) { 29676bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines llvm::Type *OTy = Op->getType(); 29686bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 29696bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // FIXME: this is utterly horrific. We should not be looking at previous 29706bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // codegen context to find out what needs doing. Unfortunately TableGen 29716bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // currently gives us exactly the same calls for vceqz_f32 and vceqz_s32 29726bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // (etc). 29736bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (BitCastInst *BI = dyn_cast<BitCastInst>(Op)) 29746bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines OTy = BI->getOperand(0)->getType(); 29756bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 29766bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Op = Builder.CreateBitCast(Op, OTy); 29776bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (OTy->getScalarType()->isFloatingPointTy()) { 29786bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Op = Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy)); 29796bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } else { 29806bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Op = Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy)); 2981651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 29826bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return Builder.CreateSExt(Op, Ty, Name); 29836bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines} 29846bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 29856bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hinesstatic Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops, 29866bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *ExtOp, Value *IndexOp, 29876bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines llvm::Type *ResTy, unsigned IntID, 29886bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines const char *Name) { 29896bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SmallVector<Value *, 2> TblOps; 29906bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (ExtOp) 29916bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines TblOps.push_back(ExtOp); 29926bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 29936bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // Build a vector containing sequential number like (0, 1, 2, ..., 15) 29946bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SmallVector<Constant*, 16> Indices; 29956bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines llvm::VectorType *TblTy = cast<llvm::VectorType>(Ops[0]->getType()); 29966bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines for (unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) { 29976bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Indices.push_back(ConstantInt::get(CGF.Int32Ty, 2*i)); 29986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Indices.push_back(ConstantInt::get(CGF.Int32Ty, 2*i+1)); 2999651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 30006bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *SV = llvm::ConstantVector::get(Indices); 30016bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 30026bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines int PairPos = 0, End = Ops.size() - 1; 30036bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines while (PairPos < End) { 30046bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], 30056bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[PairPos+1], SV, Name)); 30066bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines PairPos += 2; 3007651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 30086bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 30096bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // If there's an odd number of 64-bit lookup table, fill the high 64-bit 30106bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // of the 128-bit lookup table with zero. 30116bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (PairPos == End) { 30126bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *ZeroTbl = ConstantAggregateZero::get(TblTy); 30136bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], 30146bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ZeroTbl, SV, Name)); 3015651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 30166bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 30176bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *TblF; 30186bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines TblOps.push_back(IndexOp); 30196bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines TblF = CGF.CGM.getIntrinsic(IntID, ResTy); 30206bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 30216bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return CGF.EmitNeonCall(TblF, TblOps, Name); 3022651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 3023651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3024651f13cea278ec967336033dd032faef0e9fc2ecStephen HinesValue *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, 3025651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const CallExpr *E) { 30266bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines unsigned HintID = static_cast<unsigned>(-1); 30276bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines switch (BuiltinID) { 30286bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines default: break; 30296bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case ARM::BI__yield: 30306bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines HintID = 1; 30316bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines break; 30326bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case ARM::BI__wfe: 30336bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines HintID = 2; 30346bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines break; 30356bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case ARM::BI__wfi: 30366bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines HintID = 3; 30376bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines break; 30386bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case ARM::BI__sev: 30396bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines HintID = 4; 30406bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines break; 30416bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case ARM::BI__sevl: 30426bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines HintID = 5; 30436bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines break; 30446bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } 30456bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 30466bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (HintID != static_cast<unsigned>(-1)) { 30476bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::arm_hint); 30486bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID)); 30496bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } 30506bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 3051651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (BuiltinID == ARM::BI__clear_cache) { 3052651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments"); 3053651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const FunctionDecl *FD = E->getDirectCallee(); 3054651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value*, 2> Ops; 3055651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines for (unsigned i = 0; i < 2; i++) 3056651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(i))); 3057651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType()); 3058651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty); 3059651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines StringRef Name = FD->getName(); 3060651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); 3061651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3062651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3063651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (BuiltinID == ARM::BI__builtin_arm_ldrexd || 3064651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines (BuiltinID == ARM::BI__builtin_arm_ldrex && 3065651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines getContext().getTypeSize(E->getType()) == 64)) { 3066651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(Intrinsic::arm_ldrexd); 3067651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3068651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *LdPtr = EmitScalarExpr(E->getArg(0)); 3069651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy), 3070651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "ldrexd"); 3071651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3072651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Val0 = Builder.CreateExtractValue(Val, 1); 3073651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Val1 = Builder.CreateExtractValue(Val, 0); 3074651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val0 = Builder.CreateZExt(Val0, Int64Ty); 3075651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val1 = Builder.CreateZExt(Val1, Int64Ty); 3076651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3077651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *ShiftCst = llvm::ConstantInt::get(Int64Ty, 32); 3078651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */); 3079651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val = Builder.CreateOr(Val, Val1); 3080651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateBitCast(Val, ConvertType(E->getType())); 3081651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3082651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3083651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (BuiltinID == ARM::BI__builtin_arm_ldrex) { 3084651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *LoadAddr = EmitScalarExpr(E->getArg(0)); 3085651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3086651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines QualType Ty = E->getType(); 3087651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *RealResTy = ConvertType(Ty); 3088651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *IntResTy = llvm::IntegerType::get(getLLVMContext(), 3089651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines getContext().getTypeSize(Ty)); 3090651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines LoadAddr = Builder.CreateBitCast(LoadAddr, IntResTy->getPointerTo()); 3091651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3092651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(Intrinsic::arm_ldrex, LoadAddr->getType()); 3093651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Val = Builder.CreateCall(F, LoadAddr, "ldrex"); 3094651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3095651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (RealResTy->isPointerTy()) 3096651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateIntToPtr(Val, RealResTy); 3097651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines else { 3098651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val = Builder.CreateTruncOrBitCast(Val, IntResTy); 3099651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateBitCast(Val, RealResTy); 3100651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3101651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3102651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3103651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (BuiltinID == ARM::BI__builtin_arm_strexd || 3104651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines (BuiltinID == ARM::BI__builtin_arm_strex && 3105651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines getContext().getTypeSize(E->getArg(0)->getType()) == 64)) { 3106651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(Intrinsic::arm_strexd); 3107651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, NULL); 3108651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3109651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Tmp = CreateMemTemp(E->getArg(0)->getType()); 3110651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Val = EmitScalarExpr(E->getArg(0)); 3111651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Builder.CreateStore(Val, Tmp); 3112651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3113651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy)); 3114651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val = Builder.CreateLoad(LdPtr); 3115651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3116651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Arg0 = Builder.CreateExtractValue(Val, 0); 3117651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Arg1 = Builder.CreateExtractValue(Val, 1); 3118651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), Int8PtrTy); 3119651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall3(F, Arg0, Arg1, StPtr, "strexd"); 31202e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu } 3121651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3122651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (BuiltinID == ARM::BI__builtin_arm_strex) { 3123651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *StoreVal = EmitScalarExpr(E->getArg(0)); 3124651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *StoreAddr = EmitScalarExpr(E->getArg(1)); 3125651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3126651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines QualType Ty = E->getArg(0)->getType(); 3127651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(), 3128651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines getContext().getTypeSize(Ty)); 3129651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo()); 3130651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3131651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (StoreVal->getType()->isPointerTy()) 3132651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines StoreVal = Builder.CreatePtrToInt(StoreVal, Int32Ty); 3133651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines else { 3134651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines StoreVal = Builder.CreateBitCast(StoreVal, StoreTy); 3135651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int32Ty); 3136651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3137651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3138651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(Intrinsic::arm_strex, StoreAddr->getType()); 3139651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall2(F, StoreVal, StoreAddr, "strex"); 31402e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu } 3141651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3142651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (BuiltinID == ARM::BI__builtin_arm_clrex) { 3143651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(Intrinsic::arm_clrex); 3144651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall(F); 31452e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu } 31462e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu 3147651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // CRC32 3148651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic; 3149651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 3150651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case ARM::BI__builtin_arm_crc32b: 3151651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CRCIntrinsicID = Intrinsic::arm_crc32b; break; 3152651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case ARM::BI__builtin_arm_crc32cb: 3153651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CRCIntrinsicID = Intrinsic::arm_crc32cb; break; 3154651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case ARM::BI__builtin_arm_crc32h: 3155651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CRCIntrinsicID = Intrinsic::arm_crc32h; break; 3156651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case ARM::BI__builtin_arm_crc32ch: 3157651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CRCIntrinsicID = Intrinsic::arm_crc32ch; break; 3158651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case ARM::BI__builtin_arm_crc32w: 3159651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case ARM::BI__builtin_arm_crc32d: 3160651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CRCIntrinsicID = Intrinsic::arm_crc32w; break; 3161651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case ARM::BI__builtin_arm_crc32cw: 3162651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case ARM::BI__builtin_arm_crc32cd: 3163651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CRCIntrinsicID = Intrinsic::arm_crc32cw; break; 3164651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 31652e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu 3166651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (CRCIntrinsicID != Intrinsic::not_intrinsic) { 3167651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Arg0 = EmitScalarExpr(E->getArg(0)); 3168651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Arg1 = EmitScalarExpr(E->getArg(1)); 3169aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu 3170651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // crc32{c,}d intrinsics are implemnted as two calls to crc32{c,}w 3171651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // intrinsics, hence we need different codegen for these cases. 3172651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (BuiltinID == ARM::BI__builtin_arm_crc32d || 3173651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines BuiltinID == ARM::BI__builtin_arm_crc32cd) { 3174651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *C1 = llvm::ConstantInt::get(Int64Ty, 32); 3175651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Arg1a = Builder.CreateTruncOrBitCast(Arg1, Int32Ty); 3176651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Arg1b = Builder.CreateLShr(Arg1, C1); 3177651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Arg1b = Builder.CreateTruncOrBitCast(Arg1b, Int32Ty); 31782e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu 3179651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(CRCIntrinsicID); 3180651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Res = Builder.CreateCall2(F, Arg0, Arg1a); 3181651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall2(F, Res, Arg1b); 3182651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } else { 3183651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Arg1 = Builder.CreateZExtOrBitCast(Arg1, Int32Ty); 3184ff920eec4d449bee560d8d99636ad0eb50cd9d8dTim Northover 3185651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(CRCIntrinsicID); 3186651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall2(F, Arg0, Arg1); 3187651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3188ff920eec4d449bee560d8d99636ad0eb50cd9d8dTim Northover } 3189ff920eec4d449bee560d8d99636ad0eb50cd9d8dTim Northover 3190651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value*, 4> Ops; 31916bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines llvm::Value *Align = nullptr; 3192b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) { 3193dd12780e86575795fa912529a911b01e2abc4677Hao Liu if (i == 0) { 3194dd12780e86575795fa912529a911b01e2abc4677Hao Liu switch (BuiltinID) { 3195651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_v: 3196651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_v: 3197651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_lane_v: 3198651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_lane_v: 3199651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_dup_v: 3200651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_dup_v: 3201651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_v: 3202651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_v: 3203651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_lane_v: 3204651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_lane_v: 3205651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2_v: 3206651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2q_v: 3207651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2_lane_v: 3208651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2q_lane_v: 3209651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3_v: 3210651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3q_v: 3211651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3_lane_v: 3212651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3q_lane_v: 3213651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4_v: 3214651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4q_v: 3215651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4_lane_v: 3216651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4q_lane_v: 3217dd12780e86575795fa912529a911b01e2abc4677Hao Liu // Get the alignment for the argument in addition to the value; 3218dd12780e86575795fa912529a911b01e2abc4677Hao Liu // we'll use it later. 3219651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines std::pair<llvm::Value*, unsigned> Src = 3220dd12780e86575795fa912529a911b01e2abc4677Hao Liu EmitPointerWithAlignment(E->getArg(0)); 3221dd12780e86575795fa912529a911b01e2abc4677Hao Liu Ops.push_back(Src.first); 3222dd12780e86575795fa912529a911b01e2abc4677Hao Liu Align = Builder.getInt32(Src.second); 3223dd12780e86575795fa912529a911b01e2abc4677Hao Liu continue; 3224dd12780e86575795fa912529a911b01e2abc4677Hao Liu } 3225dd12780e86575795fa912529a911b01e2abc4677Hao Liu } 3226dd12780e86575795fa912529a911b01e2abc4677Hao Liu if (i == 1) { 3227dd12780e86575795fa912529a911b01e2abc4677Hao Liu switch (BuiltinID) { 3228651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_v: 3229651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2q_v: 3230651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_v: 3231651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3q_v: 3232651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_v: 3233651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4q_v: 3234651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_lane_v: 3235651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2q_lane_v: 3236651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_lane_v: 3237651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3q_lane_v: 3238651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_lane_v: 3239651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4q_lane_v: 3240651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_dup_v: 3241651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_dup_v: 3242651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_dup_v: 3243dd12780e86575795fa912529a911b01e2abc4677Hao Liu // Get the alignment for the argument in addition to the value; 3244dd12780e86575795fa912529a911b01e2abc4677Hao Liu // we'll use it later. 3245651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines std::pair<llvm::Value*, unsigned> Src = 3246dd12780e86575795fa912529a911b01e2abc4677Hao Liu EmitPointerWithAlignment(E->getArg(1)); 3247dd12780e86575795fa912529a911b01e2abc4677Hao Liu Ops.push_back(Src.first); 3248dd12780e86575795fa912529a911b01e2abc4677Hao Liu Align = Builder.getInt32(Src.second); 3249dd12780e86575795fa912529a911b01e2abc4677Hao Liu continue; 3250dd12780e86575795fa912529a911b01e2abc4677Hao Liu } 3251dd12780e86575795fa912529a911b01e2abc4677Hao Liu } 3252b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover Ops.push_back(EmitScalarExpr(E->getArg(i))); 3253b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover } 3254b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover 3255651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 3256651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines default: break; 3257651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // vget_lane and vset_lane are not overloaded and do not have an extra 3258651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // argument that specifies the vector type. 3259651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_i8: 3260651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_i16: 3261651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_i32: 3262651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_i64: 3263651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_f32: 3264651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_i8: 3265651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_i16: 3266651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_i32: 3267651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_i64: 3268651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_f32: 3269651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 3270651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vget_lane"); 3271651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_i8: 3272651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_i16: 3273651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_i32: 3274651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_i64: 3275651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_f32: 3276651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_i8: 3277651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_i16: 3278651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_i32: 3279651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_i64: 3280651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_f32: 3281651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(2))); 3282651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 3283651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3284651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Non-polymorphic crypto instructions also not overloaded 3285651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsha1h_u32: 3286651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 3287651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops, 3288651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vsha1h"); 3289651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsha1cq_u32: 3290651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(2))); 3291651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops, 3292651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vsha1h"); 3293651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsha1pq_u32: 3294651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(2))); 3295651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops, 3296651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vsha1h"); 3297651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsha1mq_u32: 3298651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(2))); 3299651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops, 3300651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vsha1h"); 3301651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3302651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3303b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover // Get the last argument, which specifies the vector type. 3304b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover llvm::APSInt Result; 3305651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const Expr *Arg = E->getArg(E->getNumArgs()-1); 3306b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover if (!Arg->isIntegerConstantExpr(Result, getContext())) 33076bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 3308b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover 3309651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f || 3310651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines BuiltinID == ARM::BI__builtin_arm_vcvtr_d) { 3311651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Determine the overloaded type of this builtin. 3312651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty; 3313651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f) 3314651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = FloatTy; 3315912502b4996b14db31b498cb1eef2b17d7d66d57Hao Liu else 3316651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = DoubleTy; 3317912502b4996b14db31b498cb1eef2b17d7d66d57Hao Liu 3318651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Determine whether this is an unsigned conversion or not. 3319651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool usgn = Result.getZExtValue() == 1; 3320651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr; 3321b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover 3322651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Call the appropriate intrinsic. 3323dd12780e86575795fa912529a911b01e2abc4677Hao Liu Function *F = CGM.getIntrinsic(Int, Ty); 3324651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall(F, Ops, "vcvtr"); 3325dd12780e86575795fa912529a911b01e2abc4677Hao Liu } 3326651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3327651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Determine the type of this overloaded NEON intrinsic. 3328651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags Type(Result.getZExtValue()); 3329651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool usgn = Type.isUnsigned(); 3330651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool rightShift = false; 3331651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3332651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType *VTy = GetNeonType(this, Type); 3333651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty = VTy; 3334651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (!Ty) 33356bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 3336651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3337651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Many NEON builtins have identical semantics and uses in ARM and 3338651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // AArch64. Emit these in a single function. 3339651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::ArrayRef<NeonIntrinsicInfo> IntrinsicMap(ARMSIMDIntrinsicMap); 3340651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap( 3341651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines IntrinsicMap, BuiltinID, NEONSIMDIntrinsicsProvenSorted); 3342651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Builtin) 3343651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitCommonNeonBuiltinExpr( 3344651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic, 3345651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Builtin->NameHint, Builtin->TypeModifier, E, Ops, Align); 3346651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3347651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Int; 3348651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 33496bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines default: return nullptr; 3350651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_lane_v: 3351651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Handle 64-bit integer elements as a special case. Use shuffles of 3352651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // one-element vectors to avoid poor code for i64 in the backend. 3353651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (VTy->getElementType()->isIntegerTy(64)) { 3354651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Extract the other lane. 3355651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3356651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines int Lane = cast<ConstantInt>(Ops[2])->getZExtValue(); 3357651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *SV = llvm::ConstantVector::get(ConstantInt::get(Int32Ty, 1-Lane)); 3358651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); 3359651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Load the value as a one-element vector. 3360651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::VectorType::get(VTy->getElementType(), 1); 3361651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Ty); 3362651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Ld = Builder.CreateCall2(F, Ops[0], Align); 3363651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Combine them. 3364651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Constant*, 2> Indices; 3365651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Indices.push_back(ConstantInt::get(Int32Ty, 1-Lane)); 3366651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Indices.push_back(ConstantInt::get(Int32Ty, Lane)); 3367651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SV = llvm::ConstantVector::get(Indices); 3368651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateShuffleVector(Ops[1], Ld, SV, "vld1q_lane"); 3369dd12780e86575795fa912529a911b01e2abc4677Hao Liu } 3370651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // fall through 3371651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_lane_v: { 337211a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 337311a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ty = llvm::PointerType::getUnqual(VTy->getElementType()); 337411a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 337511a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu LoadInst *Ld = Builder.CreateLoad(Ops[0]); 337611a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ld->setAlignment(cast<ConstantInt>(Align)->getZExtValue()); 337711a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane"); 337811a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu } 3379651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_dup_v: 3380651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_dup_v: 3381651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_dup_v: { 3382651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Handle 64-bit elements as a special-case. There is no "dup" needed. 3383651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (VTy->getElementType()->getPrimitiveSizeInBits() == 64) { 338411a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu switch (BuiltinID) { 3385651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_dup_v: 338611a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Int = Intrinsic::arm_neon_vld2; 338711a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu break; 3388651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_dup_v: 338911a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Int = Intrinsic::arm_neon_vld3; 339011a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu break; 3391651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_dup_v: 339211a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Int = Intrinsic::arm_neon_vld4; 339311a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu break; 3394651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines default: llvm_unreachable("unknown vld_dup intrinsic?"); 339511a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu } 339611a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Function *F = CGM.getIntrinsic(Int, Ty); 339711a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld_dup"); 339811a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 339911a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 340011a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu return Builder.CreateStore(Ops[1], Ops[0]); 340111a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu } 340211a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu switch (BuiltinID) { 3403651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_dup_v: 340411a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Int = Intrinsic::arm_neon_vld2lane; 340511a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu break; 3406651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_dup_v: 340711a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Int = Intrinsic::arm_neon_vld3lane; 340811a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu break; 3409651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_dup_v: 341011a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Int = Intrinsic::arm_neon_vld4lane; 341111a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu break; 3412651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines default: llvm_unreachable("unknown vld_dup intrinsic?"); 341311a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu } 341411a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Function *F = CGM.getIntrinsic(Int, Ty); 341511a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu llvm::StructType *STy = cast<llvm::StructType>(F->getReturnType()); 341611a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu 3417651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value*, 6> Args; 341811a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Args.push_back(Ops[1]); 341911a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Args.append(STy->getNumElements(), UndefValue::get(Ty)); 342011a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu 342111a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu llvm::Constant *CI = ConstantInt::get(Int32Ty, 0); 342211a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Args.push_back(CI); 342311a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Args.push_back(Align); 342411a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu 342511a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ops[1] = Builder.CreateCall(F, Args, "vld_dup"); 342611a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu // splat lane 0 to all elts in each vector of the result. 342711a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { 342811a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Value *Val = Builder.CreateExtractValue(Ops[1], i); 342911a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Value *Elt = Builder.CreateBitCast(Val, Ty); 343011a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Elt = EmitNeonSplat(Elt, CI); 343111a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Elt = Builder.CreateBitCast(Elt, Val->getType()); 343211a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ops[1] = Builder.CreateInsertValue(Ops[1], Elt, i); 343311a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu } 343411a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 343511a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 343611a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu return Builder.CreateStore(Ops[1], Ops[0]); 343711a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu } 3438651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqrshrn_n_v: 3439651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = 3440651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns; 3441651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n", 3442651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1, true); 3443651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqrshrun_n_v: 3444651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqrshiftnsu, Ty), 3445651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vqrshrun_n", 1, true); 3446651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshlu_n_v: 3447651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshluq_n_v: 3448651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftsu, Ty), 3449651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vqshlu", 1, false); 3450651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshrn_n_v: 3451651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns; 3452651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n", 3453651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1, true); 3454651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshrun_n_v: 3455651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftnsu, Ty), 3456651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vqshrun_n", 1, true); 3457651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrecpe_v: 3458651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrecpeq_v: 3459651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrecpe, Ty), 3460651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vrecpe"); 3461651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrshrn_n_v: 3462651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrshiftn, Ty), 3463651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vrshrn_n", 1, true); 3464651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrshr_n_v: 3465651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrshrq_n_v: 3466651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts; 3467651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n", 1, true); 3468651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrsra_n_v: 3469651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrsraq_n_v: 34700aa1a88e19235574481e46e9e6e9ce66a9e6624fJiangning Liu Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 34710aa1a88e19235574481e46e9e6e9ce66a9e6624fJiangning Liu Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3472651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true); 3473651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts; 3474651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall2(CGM.getIntrinsic(Int, Ty), Ops[1], Ops[2]); 3475651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n"); 3476651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsri_n_v: 3477651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsriq_n_v: 3478651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines rightShift = true; 3479651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsli_n_v: 3480651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsliq_n_v: 3481651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift); 3482651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftins, Ty), 3483651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vsli_n"); 3484651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsra_n_v: 3485651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsraq_n_v: 34860aa1a88e19235574481e46e9e6e9ce66a9e6624fJiangning Liu Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3487651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); 3488651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Ops[0], Ops[1]); 3489651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_lane_v: 3490651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Handle 64-bit integer elements as a special case. Use a shuffle to get 3491651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // a one-element vector and avoid poor code for i64 in the backend. 3492651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (VTy->getElementType()->isIntegerTy(64)) { 3493651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3494651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2])); 3495651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); 3496651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Align; 3497651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1, 3498651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1]->getType()), Ops); 349951cc0172a173599b769968696e20638754d1dcd6Ana Pazos } 3500651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // fall through 3501651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_lane_v: { 3502b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3503651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); 3504651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 3505651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines StoreInst *St = Builder.CreateStore(Ops[1], 3506651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Builder.CreateBitCast(Ops[0], Ty)); 3507651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines St->setAlignment(cast<ConstantInt>(Align)->getZExtValue()); 3508651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return St; 35098137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 3510651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl1_v: 3511651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl1), 3512651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbl1"); 3513651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl2_v: 3514651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl2), 3515651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbl2"); 3516651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl3_v: 3517651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl3), 3518651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbl3"); 3519651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl4_v: 3520651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl4), 3521651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbl4"); 3522651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx1_v: 3523651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx1), 3524651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbx1"); 3525651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx2_v: 3526651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx2), 3527651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbx2"); 3528651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx3_v: 3529651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx3), 3530651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbx3"); 3531651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx4_v: 3532651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx4), 3533651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbx4"); 35348137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 3535651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 3536651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 35376bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hinesstatic Value *EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID, 3538651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const CallExpr *E, 3539651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVectorImpl<Value *> &Ops) { 3540651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned int Int = 0; 35416bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines const char *s = nullptr; 3542651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3543651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 3544651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines default: 35456bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 3546651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl1_v: 3547651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl1_v: 3548651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl1q_v: 3549651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl2_v: 3550651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl2_v: 3551651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl2q_v: 3552651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl3_v: 3553651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl3_v: 3554651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl3q_v: 3555651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl4_v: 3556651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl4_v: 3557651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl4q_v: 3558651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 3559651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx1_v: 3560651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx1_v: 3561651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx1q_v: 3562651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx2_v: 3563651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx2_v: 3564651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx2q_v: 3565651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx3_v: 3566651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx3_v: 3567651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx3q_v: 3568651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx4_v: 3569651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx4_v: 3570651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx4q_v: 3571651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 35728137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 3573651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3574651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines assert(E->getNumArgs() >= 3); 3575651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3576651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Get the last argument, which specifies the vector type. 3577651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::APSInt Result; 3578651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const Expr *Arg = E->getArg(E->getNumArgs() - 1); 3579651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (!Arg->isIntegerConstantExpr(Result, CGF.getContext())) 35806bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 3581651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3582651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Determine the type of this overloaded NEON intrinsic. 3583651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags Type(Result.getZExtValue()); 3584651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType *VTy = GetNeonType(&CGF, Type); 3585651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty = VTy; 3586651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (!Ty) 35876bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 3588651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3589651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned nElts = VTy->getNumElements(); 3590651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3591651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CodeGen::CGBuilderTy &Builder = CGF.Builder; 3592651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3593651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // AArch64 scalar builtins are not overloaded, they do not have an extra 3594651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // argument that specifies the vector type, need to handle each case. 3595651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value *, 2> TblOps; 3596651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 3597651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl1_v: { 3598651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[0]); 35996bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return packTBLDVectorList(CGF, TblOps, nullptr, Ops[1], Ty, 36006bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Intrinsic::aarch64_neon_tbl1, "vtbl1"); 36018137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 3602651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl2_v: { 3603651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[0]); 3604651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[1]); 36056bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return packTBLDVectorList(CGF, TblOps, nullptr, Ops[2], Ty, 36066bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Intrinsic::aarch64_neon_tbl1, "vtbl1"); 36078137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 3608651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl3_v: { 3609651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[0]); 3610651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[1]); 3611651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[2]); 36126bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return packTBLDVectorList(CGF, TblOps, nullptr, Ops[3], Ty, 36136bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Intrinsic::aarch64_neon_tbl2, "vtbl2"); 36148137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 3615651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl4_v: { 3616651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[0]); 3617651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[1]); 3618651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[2]); 3619651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[3]); 36206bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return packTBLDVectorList(CGF, TblOps, nullptr, Ops[4], Ty, 36216bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Intrinsic::aarch64_neon_tbl2, "vtbl2"); 36228137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 3623651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx1_v: { 3624651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[1]); 36256bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *TblRes = packTBLDVectorList(CGF, TblOps, nullptr, Ops[2], Ty, 36266bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Intrinsic::aarch64_neon_tbl1, "vtbl1"); 3627651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3628651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Constant *Eight = ConstantInt::get(VTy->getElementType(), 8); 3629651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value* EightV = llvm::ConstantVector::getSplat(nElts, Eight); 3630651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV); 3631651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CmpRes = Builder.CreateSExt(CmpRes, Ty); 3632651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3633651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); 3634651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes); 3635651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx"); 36368137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 3637651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx2_v: { 3638651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[1]); 3639651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[2]); 3640651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return packTBLDVectorList(CGF, TblOps, Ops[0], Ops[3], Ty, 36416bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Intrinsic::aarch64_neon_tbx1, "vtbx1"); 36428137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 3643651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx3_v: { 3644651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[1]); 3645651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[2]); 3646651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[3]); 36476bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *TblRes = packTBLDVectorList(CGF, TblOps, nullptr, Ops[4], Ty, 36486bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Intrinsic::aarch64_neon_tbl2, "vtbl2"); 3649651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3650651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Constant *TwentyFour = ConstantInt::get(VTy->getElementType(), 24); 3651651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value* TwentyFourV = llvm::ConstantVector::getSplat(nElts, TwentyFour); 3652651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4], 3653651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TwentyFourV); 3654651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CmpRes = Builder.CreateSExt(CmpRes, Ty); 3655651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3656651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); 3657651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes); 3658651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx"); 36598137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 3660651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx4_v: { 3661651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[1]); 3662651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[2]); 3663651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[3]); 3664651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[4]); 3665651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return packTBLDVectorList(CGF, TblOps, Ops[0], Ops[5], Ty, 36666bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Intrinsic::aarch64_neon_tbx2, "vtbx2"); 3667651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3668651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl1_v: 3669651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl1q_v: 36706bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_tbl1; s = "vtbl1"; break; 3671651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl2_v: 3672651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl2q_v: { 36736bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_tbl2; s = "vtbl2"; break; 3674651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl3_v: 3675651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl3q_v: 36766bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_tbl3; s = "vtbl3"; break; 3677651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl4_v: 3678651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl4q_v: 36796bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_tbl4; s = "vtbl4"; break; 3680651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx1_v: 3681651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx1q_v: 36826bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_tbx1; s = "vtbx1"; break; 3683651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx2_v: 3684651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx2q_v: 36856bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_tbx2; s = "vtbx2"; break; 3686651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx3_v: 3687651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx3q_v: 36886bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_tbx3; s = "vtbx3"; break; 3689651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx4_v: 3690651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx4q_v: 36916bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_tbx4; s = "vtbx4"; break; 36928137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 3693b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover } 3694651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3695651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (!Int) 36966bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 3697651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3698651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGF.CGM.getIntrinsic(Int, Ty); 3699651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return CGF.EmitNeonCall(F, Ops, s); 3700ff920eec4d449bee560d8d99636ad0eb50cd9d8dTim Northover} 3701ff920eec4d449bee560d8d99636ad0eb50cd9d8dTim Northover 3702651f13cea278ec967336033dd032faef0e9fc2ecStephen HinesValue *CodeGenFunction::vectorWrapScalar16(Value *Op) { 3703651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *VTy = llvm::VectorType::get(Int16Ty, 4); 3704651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Op = Builder.CreateBitCast(Op, Int16Ty); 3705651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *V = UndefValue::get(VTy); 3706651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Constant *CI = ConstantInt::get(Int32Ty, 0); 3707651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Op = Builder.CreateInsertElement(V, Op, CI); 3708651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Op; 3709651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 3710651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3711651f13cea278ec967336033dd032faef0e9fc2ecStephen HinesValue *CodeGenFunction::vectorWrapScalar8(Value *Op) { 3712651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *VTy = llvm::VectorType::get(Int8Ty, 8); 3713651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Op = Builder.CreateBitCast(Op, Int8Ty); 3714651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *V = UndefValue::get(VTy); 3715651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Constant *CI = ConstantInt::get(Int32Ty, 0); 3716651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Op = Builder.CreateInsertElement(V, Op, CI); 3717651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Op; 3718651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 3719651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3720651f13cea278ec967336033dd032faef0e9fc2ecStephen HinesValue *CodeGenFunction:: 3721651f13cea278ec967336033dd032faef0e9fc2ecStephen HinesemitVectorWrappedScalar8Intrinsic(unsigned Int, SmallVectorImpl<Value*> &Ops, 3722651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const char *Name) { 37236bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // i8 is not a legal types for AArch64, so we can't just use 3724651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // a normal overloaed intrinsic call for these scalar types. Instead 3725651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // we'll build 64-bit vectors w/ lane zero being our input values and 3726651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // perform the operation on that. The back end can pattern match directly 3727651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // to the scalar instruction. 3728651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = vectorWrapScalar8(Ops[0]); 3729651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = vectorWrapScalar8(Ops[1]); 3730651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *VTy = llvm::VectorType::get(Int8Ty, 8); 3731651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *V = EmitNeonCall(CGM.getIntrinsic(Int, VTy), Ops, Name); 3732651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Constant *CI = ConstantInt::get(Int32Ty, 0); 3733651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(V, CI, "lane0"); 3734651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 3735651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3736651f13cea278ec967336033dd032faef0e9fc2ecStephen HinesValue *CodeGenFunction:: 3737651f13cea278ec967336033dd032faef0e9fc2ecStephen HinesemitVectorWrappedScalar16Intrinsic(unsigned Int, SmallVectorImpl<Value*> &Ops, 3738651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const char *Name) { 37396bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // i16 is not a legal types for AArch64, so we can't just use 3740651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // a normal overloaed intrinsic call for these scalar types. Instead 3741651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // we'll build 64-bit vectors w/ lane zero being our input values and 3742651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // perform the operation on that. The back end can pattern match directly 3743651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // to the scalar instruction. 3744651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = vectorWrapScalar16(Ops[0]); 3745651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = vectorWrapScalar16(Ops[1]); 3746651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *VTy = llvm::VectorType::get(Int16Ty, 4); 3747651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *V = EmitNeonCall(CGM.getIntrinsic(Int, VTy), Ops, Name); 3748651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Constant *CI = ConstantInt::get(Int32Ty, 0); 3749651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(V, CI, "lane0"); 3750651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 3751651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 37526bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen HinesValue *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, 37536bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines const CallExpr *E) { 37546bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (BuiltinID == AArch64::BI__clear_cache) { 37554537d6e0f9baf2e011a4260e0d7872789b01c3f2Rafael Espindola assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments"); 375679ba509b0106fd0a1ff832baeb1fdb5430527efeRafael Espindola const FunctionDecl *FD = E->getDirectCallee(); 37575f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner SmallVector<Value*, 2> Ops; 37584537d6e0f9baf2e011a4260e0d7872789b01c3f2Rafael Espindola for (unsigned i = 0; i < 2; i++) 37598a37c79f03e62aecfa2c58b9179f90dd1fcdb253Eric Christopher Ops.push_back(EmitScalarExpr(E->getArg(i))); 37602acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType()); 37612acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty); 37625f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner StringRef Name = FD->getName(); 3763bd7370a78604e9a20d698bfe328c1e43f12a0613John McCall return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); 37642752c0137d95aa2f4ee1cdff4b564bac842e041bChris Lattner } 3765e140af3e27016f902146023fba7680b43043ec07Rafael Espindola 37666bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (BuiltinID == AArch64::BI__builtin_arm_ldrex && 3767651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines getContext().getTypeSize(E->getType()) == 128) { 37686bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_ldxp); 376926c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes 377026c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes Value *LdPtr = EmitScalarExpr(E->getArg(0)); 377109df2b066221d869f17f4b5762405f111a65f983Tim Northover Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy), 3772651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "ldxp"); 377326c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes 377426c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes Value *Val0 = Builder.CreateExtractValue(Val, 1); 377526c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes Value *Val1 = Builder.CreateExtractValue(Val, 0); 3776651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128); 3777651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val0 = Builder.CreateZExt(Val0, Int128Ty); 3778651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val1 = Builder.CreateZExt(Val1, Int128Ty); 377926c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes 3780651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64); 378126c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */); 378209df2b066221d869f17f4b5762405f111a65f983Tim Northover Val = Builder.CreateOr(Val, Val1); 378309df2b066221d869f17f4b5762405f111a65f983Tim Northover return Builder.CreateBitCast(Val, ConvertType(E->getType())); 37846bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } else if (BuiltinID == AArch64::BI__builtin_arm_ldrex) { 378509df2b066221d869f17f4b5762405f111a65f983Tim Northover Value *LoadAddr = EmitScalarExpr(E->getArg(0)); 378609df2b066221d869f17f4b5762405f111a65f983Tim Northover 378709df2b066221d869f17f4b5762405f111a65f983Tim Northover QualType Ty = E->getType(); 378809df2b066221d869f17f4b5762405f111a65f983Tim Northover llvm::Type *RealResTy = ConvertType(Ty); 378909df2b066221d869f17f4b5762405f111a65f983Tim Northover llvm::Type *IntResTy = llvm::IntegerType::get(getLLVMContext(), 379009df2b066221d869f17f4b5762405f111a65f983Tim Northover getContext().getTypeSize(Ty)); 379109df2b066221d869f17f4b5762405f111a65f983Tim Northover LoadAddr = Builder.CreateBitCast(LoadAddr, IntResTy->getPointerTo()); 379209df2b066221d869f17f4b5762405f111a65f983Tim Northover 37936bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_ldxr, LoadAddr->getType()); 3794651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Val = Builder.CreateCall(F, LoadAddr, "ldxr"); 379509df2b066221d869f17f4b5762405f111a65f983Tim Northover 379609df2b066221d869f17f4b5762405f111a65f983Tim Northover if (RealResTy->isPointerTy()) 379709df2b066221d869f17f4b5762405f111a65f983Tim Northover return Builder.CreateIntToPtr(Val, RealResTy); 3798651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3799651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val = Builder.CreateTruncOrBitCast(Val, IntResTy); 3800651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateBitCast(Val, RealResTy); 380109df2b066221d869f17f4b5762405f111a65f983Tim Northover } 380209df2b066221d869f17f4b5762405f111a65f983Tim Northover 38036bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (BuiltinID == AArch64::BI__builtin_arm_strex && 3804651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines getContext().getTypeSize(E->getArg(0)->getType()) == 128) { 38056bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_stxp); 3806651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *STy = llvm::StructType::get(Int64Ty, Int64Ty, NULL); 380726c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes 3808651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *One = llvm::ConstantInt::get(Int32Ty, 1); 3809651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Tmp = Builder.CreateAlloca(ConvertType(E->getArg(0)->getType()), 3810651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines One); 381126c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes Value *Val = EmitScalarExpr(E->getArg(0)); 381226c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes Builder.CreateStore(Val, Tmp); 381326c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes 381426c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes Value *LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy)); 381526c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes Val = Builder.CreateLoad(LdPtr); 381626c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes 381726c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes Value *Arg0 = Builder.CreateExtractValue(Val, 0); 381826c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes Value *Arg1 = Builder.CreateExtractValue(Val, 1); 3819651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), 3820651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int8PtrTy); 3821651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall3(F, Arg0, Arg1, StPtr, "stxp"); 38226bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } else if (BuiltinID == AArch64::BI__builtin_arm_strex) { 382309df2b066221d869f17f4b5762405f111a65f983Tim Northover Value *StoreVal = EmitScalarExpr(E->getArg(0)); 382409df2b066221d869f17f4b5762405f111a65f983Tim Northover Value *StoreAddr = EmitScalarExpr(E->getArg(1)); 382509df2b066221d869f17f4b5762405f111a65f983Tim Northover 382609df2b066221d869f17f4b5762405f111a65f983Tim Northover QualType Ty = E->getArg(0)->getType(); 382709df2b066221d869f17f4b5762405f111a65f983Tim Northover llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(), 382809df2b066221d869f17f4b5762405f111a65f983Tim Northover getContext().getTypeSize(Ty)); 382909df2b066221d869f17f4b5762405f111a65f983Tim Northover StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo()); 383009df2b066221d869f17f4b5762405f111a65f983Tim Northover 383109df2b066221d869f17f4b5762405f111a65f983Tim Northover if (StoreVal->getType()->isPointerTy()) 3832651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines StoreVal = Builder.CreatePtrToInt(StoreVal, Int64Ty); 383309df2b066221d869f17f4b5762405f111a65f983Tim Northover else { 383409df2b066221d869f17f4b5762405f111a65f983Tim Northover StoreVal = Builder.CreateBitCast(StoreVal, StoreTy); 3835651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int64Ty); 383609df2b066221d869f17f4b5762405f111a65f983Tim Northover } 383709df2b066221d869f17f4b5762405f111a65f983Tim Northover 38386bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_stxr, StoreAddr->getType()); 3839651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall2(F, StoreVal, StoreAddr, "stxr"); 384009df2b066221d869f17f4b5762405f111a65f983Tim Northover } 384109df2b066221d869f17f4b5762405f111a65f983Tim Northover 38426bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (BuiltinID == AArch64::BI__builtin_arm_clrex) { 38436bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_clrex); 3844594e193073fe36b9ecbce9a105c269e20f82b3c9Joey Gouly return Builder.CreateCall(F); 3845594e193073fe36b9ecbce9a105c269e20f82b3c9Joey Gouly } 3846594e193073fe36b9ecbce9a105c269e20f82b3c9Joey Gouly 3847520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly // CRC32 3848520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic; 3849520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly switch (BuiltinID) { 38506bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case AArch64::BI__builtin_arm_crc32b: 38516bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines CRCIntrinsicID = Intrinsic::aarch64_crc32b; break; 38526bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case AArch64::BI__builtin_arm_crc32cb: 38536bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines CRCIntrinsicID = Intrinsic::aarch64_crc32cb; break; 38546bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case AArch64::BI__builtin_arm_crc32h: 38556bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines CRCIntrinsicID = Intrinsic::aarch64_crc32h; break; 38566bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case AArch64::BI__builtin_arm_crc32ch: 38576bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines CRCIntrinsicID = Intrinsic::aarch64_crc32ch; break; 38586bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case AArch64::BI__builtin_arm_crc32w: 38596bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines CRCIntrinsicID = Intrinsic::aarch64_crc32w; break; 38606bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case AArch64::BI__builtin_arm_crc32cw: 38616bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines CRCIntrinsicID = Intrinsic::aarch64_crc32cw; break; 38626bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case AArch64::BI__builtin_arm_crc32d: 38636bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines CRCIntrinsicID = Intrinsic::aarch64_crc32x; break; 38646bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case AArch64::BI__builtin_arm_crc32cd: 38656bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines CRCIntrinsicID = Intrinsic::aarch64_crc32cx; break; 3866520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly } 3867520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly 3868520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly if (CRCIntrinsicID != Intrinsic::not_intrinsic) { 3869520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly Value *Arg0 = EmitScalarExpr(E->getArg(0)); 3870520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly Value *Arg1 = EmitScalarExpr(E->getArg(1)); 3871651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(CRCIntrinsicID); 3872520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly 3873651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *DataTy = F->getFunctionType()->getParamType(1); 3874651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Arg1 = Builder.CreateZExtOrBitCast(Arg1, DataTy); 3875520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly 3876651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall2(F, Arg0, Arg1); 3877520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly } 3878520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly 3879651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::SmallVector<Value*, 4> Ops; 3880651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) 3881e140af3e27016f902146023fba7680b43043ec07Rafael Espindola Ops.push_back(EmitScalarExpr(E->getArg(i))); 3882651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 38836bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines llvm::ArrayRef<NeonIntrinsicInfo> SISDMap(AArch64SISDIntrinsicMap); 3884651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap( 38856bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SISDMap, BuiltinID, AArch64SISDIntrinsicsProvenSorted); 3886651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3887651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Builtin) { 3888651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(E->getNumArgs() - 1))); 3889651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Result = EmitCommonNeonSISDBuiltinExpr(*this, *Builtin, Ops, E); 3890651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines assert(Result && "SISD intrinsic should have been handled"); 3891651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Result; 3892ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman } 3893e140af3e27016f902146023fba7680b43043ec07Rafael Espindola 3894651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::APSInt Result; 3895651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const Expr *Arg = E->getArg(E->getNumArgs()-1); 3896651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags Type(0); 3897651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Arg->isIntegerConstantExpr(Result, getContext())) 3898651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Determine the type of this overloaded NEON intrinsic. 3899651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Type = NeonTypeFlags(Result.getZExtValue()); 3900651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3901651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool usgn = Type.isUnsigned(); 3902651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool quad = Type.isQuad(); 3903651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3904651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Handle non-overloaded intrinsics first. 390583bbba1fc98dd3b2a351013a40808e995e648f40Bob Wilson switch (BuiltinID) { 390683bbba1fc98dd3b2a351013a40808e995e648f40Bob Wilson default: break; 3907651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vldrq_p128: { 3908651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128); 3909651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Ptr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), Int128PTy); 3910651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateLoad(Ptr); 3911651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3912651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vstrq_p128: { 3913651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128); 3914651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy); 3915651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateStore(EmitScalarExpr(E->getArg(1)), Ptr); 3916651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3917651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvts_u32_f32: 3918651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtd_u64_f64: 3919651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines usgn = true; 3920651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FALL THROUGH 3921651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvts_s32_f32: 3922651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtd_s64_f64: { 3923651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 3924651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64; 3925651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty; 3926651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *FTy = Is64 ? DoubleTy : FloatTy; 3927651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], FTy); 3928651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (usgn) 3929651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFPToUI(Ops[0], InTy); 3930651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFPToSI(Ops[0], InTy); 3931651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3932651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvts_f32_u32: 3933651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtd_f64_u64: 3934651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines usgn = true; 3935651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FALL THROUGH 3936651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvts_f32_s32: 3937651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtd_f64_s64: { 3938651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 3939651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64; 3940651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty; 3941651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *FTy = Is64 ? DoubleTy : FloatTy; 3942651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], InTy); 3943651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (usgn) 3944651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateUIToFP(Ops[0], FTy); 3945651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateSIToFP(Ops[0], FTy); 3946651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3947651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpaddd_s64: { 3948651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty = 3949651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::Type::getInt64Ty(getLLVMContext()), 2); 3950651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Vec = EmitScalarExpr(E->getArg(0)); 3951651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The vector is v2f64, so make sure it's bitcast to that. 3952651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Vec = Builder.CreateBitCast(Vec, Ty, "v2i64"); 3953651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Value *Idx0 = llvm::ConstantInt::get(Int32Ty, 0); 3954651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Value *Idx1 = llvm::ConstantInt::get(Int32Ty, 1); 3955651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0"); 3956651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1"); 3957651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Pairwise addition of a v2f64 into a scalar f64. 3958651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Op0, Op1, "vpaddd"); 3959651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3960651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpaddd_f64: { 3961651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty = 3962651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 2); 3963651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Vec = EmitScalarExpr(E->getArg(0)); 3964651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The vector is v2f64, so make sure it's bitcast to that. 3965651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Vec = Builder.CreateBitCast(Vec, Ty, "v2f64"); 3966651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Value *Idx0 = llvm::ConstantInt::get(Int32Ty, 0); 3967651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Value *Idx1 = llvm::ConstantInt::get(Int32Ty, 1); 3968651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0"); 3969651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1"); 3970651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Pairwise addition of a v2f64 into a scalar f64. 3971651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFAdd(Op0, Op1, "vpaddd"); 3972651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3973651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpadds_f32: { 3974651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty = 3975651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 2); 3976651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Vec = EmitScalarExpr(E->getArg(0)); 3977651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The vector is v2f32, so make sure it's bitcast to that. 3978651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Vec = Builder.CreateBitCast(Vec, Ty, "v2f32"); 3979651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Value *Idx0 = llvm::ConstantInt::get(Int32Ty, 0); 3980651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Value *Idx1 = llvm::ConstantInt::get(Int32Ty, 1); 3981651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0"); 3982651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1"); 3983651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Pairwise addition of a v2f32 into a scalar f32. 3984651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFAdd(Op0, Op1, "vpaddd"); 3985651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3986651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqzd_s64: 3987651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqzd_f64: 3988651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqzs_f32: 3989651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 3990651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr( 3991651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OEQ, 3992651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ICmpInst::ICMP_EQ, "vceqz"); 3993651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgezd_s64: 3994651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgezd_f64: 3995651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgezs_f32: 3996651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 3997651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr( 3998651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OGE, 3999651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ICmpInst::ICMP_SGE, "vcgez"); 4000651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclezd_s64: 4001651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclezd_f64: 4002651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclezs_f32: 4003651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4004651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr( 4005651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OLE, 4006651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ICmpInst::ICMP_SLE, "vclez"); 4007651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtzd_s64: 4008651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtzd_f64: 4009651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtzs_f32: 4010651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4011651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr( 4012651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OGT, 4013651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ICmpInst::ICMP_SGT, "vcgtz"); 4014651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltzd_s64: 4015651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltzd_f64: 4016651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltzs_f32: 4017651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4018651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr( 4019651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OLT, 4020651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ICmpInst::ICMP_SLT, "vcltz"); 4021651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4022651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqzd_u64: { 4023651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty = llvm::Type::getInt64Ty(getLLVMContext()); 4024651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4025651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4026651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateICmp(llvm::ICmpInst::ICMP_EQ, Ops[0], 4027651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Constant::getNullValue(Ty)); 4028651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateSExt(Ops[0], Ty, "vceqzd"); 4029651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4030651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqd_f64: 4031651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_f64: 4032651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_f64: 4033651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcged_f64: 4034651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtd_f64: { 4035651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::CmpInst::Predicate P; 4036651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 4037651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines default: llvm_unreachable("missing builtin ID in switch!"); 4038651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqd_f64: P = llvm::FCmpInst::FCMP_OEQ; break; 4039651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_f64: P = llvm::FCmpInst::FCMP_OLE; break; 4040651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_f64: P = llvm::FCmpInst::FCMP_OLT; break; 4041651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcged_f64: P = llvm::FCmpInst::FCMP_OGE; break; 4042651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtd_f64: P = llvm::FCmpInst::FCMP_OGT; break; 4043651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4044651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 4045651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 4046651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); 4047651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); 4048651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd"); 4049651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4050651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqs_f32: 4051651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcles_f32: 4052651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclts_f32: 4053651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcges_f32: 4054651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgts_f32: { 4055651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::CmpInst::Predicate P; 4056651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 4057651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines default: llvm_unreachable("missing builtin ID in switch!"); 4058651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqs_f32: P = llvm::FCmpInst::FCMP_OEQ; break; 4059651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcles_f32: P = llvm::FCmpInst::FCMP_OLE; break; 4060651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclts_f32: P = llvm::FCmpInst::FCMP_OLT; break; 4061651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcges_f32: P = llvm::FCmpInst::FCMP_OGE; break; 4062651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgts_f32: P = llvm::FCmpInst::FCMP_OGT; break; 4063651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4064651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 4065651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy); 4066651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy); 4067651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); 4068651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd"); 4069651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4070651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqd_s64: 4071651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqd_u64: 4072651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtd_s64: 4073651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtd_u64: 4074651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_s64: 4075651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_u64: 4076651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcged_u64: 4077651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcged_s64: 4078651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_u64: 4079651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_s64: { 4080651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::CmpInst::Predicate P; 4081651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 4082651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines default: llvm_unreachable("missing builtin ID in switch!"); 4083651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqd_s64: 4084651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqd_u64:P = llvm::ICmpInst::ICMP_EQ;break; 4085651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtd_s64:P = llvm::ICmpInst::ICMP_SGT;break; 4086651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtd_u64:P = llvm::ICmpInst::ICMP_UGT;break; 4087651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_s64:P = llvm::ICmpInst::ICMP_SLT;break; 4088651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_u64:P = llvm::ICmpInst::ICMP_ULT;break; 4089651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcged_u64:P = llvm::ICmpInst::ICMP_UGE;break; 4090651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcged_s64:P = llvm::ICmpInst::ICMP_SGE;break; 4091651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_u64:P = llvm::ICmpInst::ICMP_ULE;break; 4092651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_s64:P = llvm::ICmpInst::ICMP_SLE;break; 4093651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4094651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 4095651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); 4096651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); 4097651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]); 4098651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateSExt(Ops[0], Int64Ty, "vceqd"); 4099651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4100651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtstd_s64: 4101651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtstd_u64: { 4102651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty = llvm::Type::getInt64Ty(getLLVMContext()); 4103651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 4104651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4105651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 4106651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); 4107651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], 4108651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Constant::getNullValue(Ty)); 4109651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateSExt(Ops[0], Ty, "vtstd"); 4110651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4111651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_i8: 4112651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_i16: 4113651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_i32: 4114651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_i64: 4115651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_f32: 4116651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_i8: 4117651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_i16: 4118651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_i32: 4119651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_i64: 4120651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_f32: 4121651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(2))); 4122651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 4123651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_f64: 4124651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The vector type needs a cast for the v1f64 variant. 4125651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], 4126651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(DoubleTy, 1)); 4127651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(2))); 4128651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 4129651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_f64: 4130651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The vector type needs a cast for the v2f64 variant. 4131651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], 4132651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 2)); 413383bbba1fc98dd3b2a351013a40808e995e648f40Bob Wilson Ops.push_back(EmitScalarExpr(E->getArg(2))); 413483bbba1fc98dd3b2a351013a40808e995e648f40Bob Wilson return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 413599c40bb13b523d58d7aeb6446e4f486d6918ca58Nate Begeman 4136651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_i8: 4137651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdupb_lane_i8: 4138651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 4139651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8)); 4140651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4141651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vget_lane"); 4142651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_i8: 4143651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdupb_laneq_i8: 4144651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 4145651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16)); 4146651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4147651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vgetq_lane"); 4148651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_i16: 4149651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vduph_lane_i16: 4150651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 4151651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4)); 4152651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4153651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vget_lane"); 4154651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_i16: 4155651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vduph_laneq_i16: 4156651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 4157651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8)); 4158651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4159651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vgetq_lane"); 4160651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_i32: 4161651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdups_lane_i32: 4162651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast( 4163651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0], 4164651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 32), 2)); 4165651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4166651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vget_lane"); 4167651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdups_lane_f32: 4168651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 4169651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 2)); 4170651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4171651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vdups_lane"); 4172651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_i32: 4173651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdups_laneq_i32: 4174651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 4175651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 32), 4)); 4176651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4177651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vgetq_lane"); 4178651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_i64: 4179651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdupd_lane_i64: 4180651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 4181651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 64), 1)); 4182651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4183651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vget_lane"); 4184651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdupd_lane_f64: 4185651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 4186651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 1)); 4187651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4188651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vdupd_lane"); 4189651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_i64: 4190651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdupd_laneq_i64: 4191651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 4192651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 64), 2)); 4193651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4194651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vgetq_lane"); 4195651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_f32: 4196651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 4197651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 2)); 4198651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4199651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vget_lane"); 4200651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_f64: 4201651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 4202651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 1)); 4203651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4204651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vget_lane"); 4205651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_f32: 4206651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdups_laneq_f32: 4207651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 4208651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 4)); 4209651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4210651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vgetq_lane"); 4211651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_f64: 4212651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdupd_laneq_f64: 4213651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 4214651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 2)); 4215651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4216651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vgetq_lane"); 4217651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddd_s64: 4218651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddd_u64: 4219651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->getArg(1)), "vaddd"); 4220651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsubd_s64: 4221651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsubd_u64: 4222651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateSub(Ops[0], EmitScalarExpr(E->getArg(1)), "vsubd"); 4223651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlalh_s16: 4224651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlslh_s16: { 4225651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value *, 2> ProductOps; 4226651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps.push_back(vectorWrapScalar16(Ops[1])); 4227651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps.push_back(vectorWrapScalar16(EmitScalarExpr(E->getArg(2)))); 4228651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4); 42296bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), 4230651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps, "vqdmlXl"); 4231651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Constant *CI = ConstantInt::get(Int32Ty, 0); 4232651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); 4233651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4234651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16 42356bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ? Intrinsic::aarch64_neon_sqadd 42366bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines : Intrinsic::aarch64_neon_sqsub; 4237651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops, "vqdmlXl"); 4238651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4239651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshlud_n_s64: { 4240651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 4241651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); 42426bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqshlu, Int64Ty), 42436bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops, "vqshlu_n"); 4244651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4245651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshld_n_u64: 4246651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshld_n_s64: { 4247651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64 42486bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ? Intrinsic::aarch64_neon_uqshl 42496bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines : Intrinsic::aarch64_neon_sqshl; 4250651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 4251651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); 42526bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vqshl_n"); 4253651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4254651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrshrd_n_u64: 4255651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrshrd_n_s64: { 4256651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64 42576bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ? Intrinsic::aarch64_neon_urshl 42586bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines : Intrinsic::aarch64_neon_srshl; 4259651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 42606bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines int SV = cast<ConstantInt>(Ops[1])->getSExtValue(); 42616bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[1] = ConstantInt::get(Int64Ty, -SV); 42626bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vrshr_n"); 4263651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4264651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrsrad_n_u64: 4265651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrsrad_n_s64: { 4266651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64 42676bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ? Intrinsic::aarch64_neon_urshl 42686bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines : Intrinsic::aarch64_neon_srshl; 4269651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); 4270651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2)))); 4271651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall2(CGM.getIntrinsic(Int, Int64Ty), Ops[1], 4272651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Builder.CreateSExt(Ops[2], Int64Ty)); 4273651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty)); 4274651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4275651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshld_n_s64: 4276651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshld_n_u64: { 4277651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1))); 4278651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateShl( 42796bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()), "shld_n"); 4280651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4281651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshrd_n_s64: { 4282651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1))); 4283651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAShr( 4284651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), 4285651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Amt->getZExtValue())), 42866bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines "shrd_n"); 4287651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4288651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshrd_n_u64: { 4289651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1))); 42906bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines uint64_t ShiftAmt = Amt->getZExtValue(); 42916bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // Right-shifting an unsigned value by its size yields 0. 42926bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (ShiftAmt == 64) 42936bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return ConstantInt::get(Int64Ty, 0); 42946bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt), 42956bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines "shrd_n"); 4296651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4297651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsrad_n_s64: { 4298651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2))); 4299651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateAShr( 4300651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), 4301651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Amt->getZExtValue())), 43026bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines "shrd_n"); 4303651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Ops[0], Ops[1]); 4304651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4305651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsrad_n_u64: { 4306651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2))); 43076bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines uint64_t ShiftAmt = Amt->getZExtValue(); 43086bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // Right-shifting an unsigned value by its size yields 0. 43096bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // As Op + 0 = Op, return Ops[0] directly. 43106bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (ShiftAmt == 64) 43116bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return Ops[0]; 43126bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt), 43136bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines "shrd_n"); 4314651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Ops[0], Ops[1]); 4315651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4316651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlalh_lane_s16: 4317651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlalh_laneq_s16: 4318651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlslh_lane_s16: 4319651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: { 4320651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), 4321651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "lane"); 4322651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value *, 2> ProductOps; 4323651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps.push_back(vectorWrapScalar16(Ops[1])); 4324651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps.push_back(vectorWrapScalar16(Ops[2])); 4325651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4); 43266bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), 4327651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps, "vqdmlXl"); 4328651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Constant *CI = ConstantInt::get(Int32Ty, 0); 4329651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); 4330651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.pop_back(); 4331651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4332651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 || 4333651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16) 43346bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ? Intrinsic::aarch64_neon_sqadd 43356bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines : Intrinsic::aarch64_neon_sqsub; 4336651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops, "vqdmlXl"); 4337651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4338651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlals_s32: 4339651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlsls_s32: { 4340651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value *, 2> ProductOps; 4341651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps.push_back(Ops[1]); 4342651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps.push_back(EmitScalarExpr(E->getArg(2))); 4343651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = 43446bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar), 4345651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps, "vqdmlXl"); 4346651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4347651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32 43486bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ? Intrinsic::aarch64_neon_sqadd 43496bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines : Intrinsic::aarch64_neon_sqsub; 4350651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops, "vqdmlXl"); 4351651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4352651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlals_lane_s32: 4353651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlals_laneq_s32: 4354651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlsls_lane_s32: 4355651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: { 4356651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), 4357651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "lane"); 4358651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value *, 2> ProductOps; 4359651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps.push_back(Ops[1]); 4360651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps.push_back(Ops[2]); 4361651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = 43626bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar), 4363651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps, "vqdmlXl"); 4364651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.pop_back(); 4365651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4366651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 || 4367651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32) 43686bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ? Intrinsic::aarch64_neon_sqadd 43696bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines : Intrinsic::aarch64_neon_sqsub; 4370651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl"); 4371651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 437299c40bb13b523d58d7aeb6446e4f486d6918ca58Nate Begeman } 4373e140af3e27016f902146023fba7680b43043ec07Rafael Espindola 43748b418685e9e4f02f4eb2a76e1ec063e07552b68dChris Lattner llvm::VectorType *VTy = GetNeonType(this, Type); 43759cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *Ty = VTy; 4376e140af3e27016f902146023fba7680b43043ec07Rafael Espindola if (!Ty) 43776bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 4378e140af3e27016f902146023fba7680b43043ec07Rafael Espindola 43796bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // Not all intrinsics handled by the common case work for AArch64 yet, so only 4380651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // defer to common code if it's been added to our special map. 43816bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Builtin = findNeonIntrinsicInMap(AArch64SIMDIntrinsicMap, BuiltinID, 43826bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines AArch64SIMDIntrinsicsProvenSorted); 4383651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4384651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Builtin) 4385651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitCommonNeonBuiltinExpr( 4386651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic, 43876bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Builtin->NameHint, Builtin->TypeModifier, E, Ops, nullptr); 4388651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 43896bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops)) 4390651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return V; 4391651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4392e140af3e27016f902146023fba7680b43043ec07Rafael Espindola unsigned Int; 4393e140af3e27016f902146023fba7680b43043ec07Rafael Espindola switch (BuiltinID) { 43946bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines default: return nullptr; 4395651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vbsl_v: 4396651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vbslq_v: { 4397651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *BitTy = llvm::VectorType::getInteger(VTy); 4398651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl"); 4399651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl"); 4400651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl"); 4401651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4402651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl"); 4403651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl"); 4404651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl"); 4405651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateBitCast(Ops[0], Ty); 4406651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4407651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfma_lane_v: 4408651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfmaq_lane_v: { // Only used for FP types 4409651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The ARM builtins (and instructions) have the addend as the first 4410651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // operand, but the 'fma' intrinsics have it last. Swap it around here. 4411651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Addend = Ops[0]; 4412651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Multiplicand = Ops[1]; 4413651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *LaneSource = Ops[2]; 4414651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Multiplicand; 4415651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = LaneSource; 4416651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Addend; 4417651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4418651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Now adjust things to handle the lane access. 4419651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v ? 4420651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(VTy->getElementType(), VTy->getNumElements() / 2) : 4421651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy; 4422651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Constant *cst = cast<Constant>(Ops[3]); 4423651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *SV = llvm::ConstantVector::getSplat(VTy->getNumElements(), cst); 4424651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy); 4425651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane"); 4426651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4427651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.pop_back(); 4428651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Intrinsic::fma; 4429651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmla"); 4430651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4431651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfma_laneq_v: { 4432651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType *VTy = cast<llvm::VectorType>(Ty); 4433651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // v1f64 fma should be mapped to Neon scalar f64 fma 4434651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (VTy && VTy->getElementType() == DoubleTy) { 4435651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 4436651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); 4437651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *VTy = GetNeonType(this, 4438651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags(NeonTypeFlags::Float64, false, true)); 4439651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateBitCast(Ops[2], VTy); 4440651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); 4441651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *F = CGM.getIntrinsic(Intrinsic::fma, DoubleTy); 4442651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Result = Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]); 4443651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateBitCast(Result, Ty); 4444651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4445651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 4446651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4447651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 44486bf1e8eadd36f077cf5403d23c664129d98eaf79Tim Northover 4449651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *STy = llvm::VectorType::get(VTy->getElementType(), 4450651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy->getNumElements() * 2); 4451651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateBitCast(Ops[2], STy); 4452651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value* SV = llvm::ConstantVector::getSplat(VTy->getNumElements(), 4453651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines cast<ConstantInt>(Ops[3])); 4454651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane"); 44556bf1e8eadd36f077cf5403d23c664129d98eaf79Tim Northover 4456651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall3(F, Ops[2], Ops[1], Ops[0]); 4457651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4458651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfmaq_laneq_v: { 4459651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 4460651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4461651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 44626bf1e8eadd36f077cf5403d23c664129d98eaf79Tim Northover 4463651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 4464651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3])); 4465651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall3(F, Ops[2], Ops[1], Ops[0]); 44666bf1e8eadd36f077cf5403d23c664129d98eaf79Tim Northover } 4467651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfmas_lane_f32: 4468651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfmas_laneq_f32: 4469651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfmad_lane_f64: 4470651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfmad_laneq_f64: { 4471651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(3))); 4472651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty = ConvertType(E->getCallReturnType()); 4473651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 4474651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); 4475651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]); 447630d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman } 4477651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfms_v: 4478651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfmsq_v: { // Only used for FP types 4479651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: probably remove when we no longer support aarch64_simd.h 4480651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // (arm_neon.h delegates to vfma). 4481651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4482651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The ARM builtins (and instructions) have the addend as the first 4483651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // operand, but the 'fma' intrinsics have it last. Swap it around here. 4484651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Subtrahend = Ops[0]; 4485651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Multiplicand = Ops[2]; 4486651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Multiplicand; 4487651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Subtrahend; 4488651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], VTy); 4489651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateFNeg(Ops[1]); 4490651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Intrinsic::fma; 4491651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmls"); 449230d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman } 4493651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmull_v: 4494651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 44956bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull; 44966bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (Type.isPoly()) Int = Intrinsic::aarch64_neon_pmull; 4497651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); 4498651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmax_v: 4499651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxq_v: 4500651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 45016bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax; 45026bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax; 4503651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax"); 4504651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmin_v: 4505651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminq_v: 4506651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 45076bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin; 45086bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin; 4509651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin"); 4510651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vabd_v: 4511651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vabdq_v: 4512651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 45136bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd; 45146bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd; 4515651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd"); 4516651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpadal_v: 4517651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpadalq_v: { 4518651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned ArgElts = VTy->getNumElements(); 4519651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType()); 4520651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned BitWidth = EltTy->getBitWidth(); 4521651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *ArgTy = llvm::VectorType::get( 4522651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), BitWidth/2), 2*ArgElts); 4523651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type* Tys[2] = { VTy, ArgTy }; 45246bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp; 4525651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<llvm::Value*, 1> TmpOps; 4526651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TmpOps.push_back(Ops[1]); 4527651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(Int, Tys); 4528651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vpadal"); 4529651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType()); 4530651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(tmp, addend); 4531651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4532651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpmin_v: 4533651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpminq_v: 4534651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 45356bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp; 45366bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp; 4537651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin"); 4538651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpmax_v: 4539651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpmaxq_v: 4540651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 45416bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp; 45426bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp; 4543651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax"); 4544651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminnm_v: 4545651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminnmq_v: 45466bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_fminnm; 4547651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vminnm"); 4548651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxnm_v: 4549651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxnmq_v: 45506bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_fmaxnm; 4551651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm"); 4552651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrecpss_f32: { 4553651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *f32Type = llvm::Type::getFloatTy(getLLVMContext()); 4554651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 45556bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, f32Type), 4556651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vrecps"); 455730d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman } 4558651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrecpsd_f64: { 4559651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *f64Type = llvm::Type::getDoubleTy(getLLVMContext()); 4560651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 45616bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, f64Type), 4562651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vrecps"); 4563651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4564651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrshr_n_v: 4565651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrshrq_n_v: 4566651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: this can be shared with 32-bit ARM, but not AArch64 at the 4567651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // moment. After the final merge it should be added to 4568651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // EmitCommonNeonBuiltinExpr. 45696bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl; 4570651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n", 1, true); 4571651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshlu_n_v: 4572651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshluq_n_v: 4573651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: AArch64 and ARM use different intrinsics for this, but are 4574651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // essentially compatible. It should be in EmitCommonNeonBuiltinExpr after 4575651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // the final merge. 45766bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_sqshlu; 4577651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshlu_n", 1, false); 4578651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshrun_n_v: 4579651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: as above 45806bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_sqshrun; 4581651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrun_n"); 4582651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqrshrun_n_v: 4583651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: and again. 45846bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_sqrshrun; 4585651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrun_n"); 4586651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshrn_n_v: 4587651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: guess 45886bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn; 4589651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n"); 4590651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrshrn_n_v: 4591651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: there might be a pattern here. 45926bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_rshrn; 4593651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshrn_n"); 4594651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqrshrn_n_v: 4595651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: another one 45966bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn; 4597651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n"); 4598651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrnda_v: 4599651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndaq_v: { 4600651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Intrinsic::round; 4601651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnda"); 4602651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4603651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndi_v: 4604651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndiq_v: { 4605651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Intrinsic::nearbyint; 4606651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndi"); 4607651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4608651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndm_v: 4609651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndmq_v: { 4610651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Intrinsic::floor; 4611651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndm"); 4612651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4613651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndn_v: 4614651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndnq_v: { 46156bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_frintn; 4616651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn"); 4617651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4618651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndp_v: 4619651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndpq_v: { 4620651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Intrinsic::ceil; 4621651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndp"); 4622651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4623651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndx_v: 4624651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndxq_v: { 4625651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Intrinsic::rint; 4626651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndx"); 4627651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4628651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrnd_v: 4629651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndq_v: { 4630651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Intrinsic::trunc; 4631651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndz"); 4632651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4633651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqz_v: 4634651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqzq_v: 4635651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OEQ, 4636651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ICmpInst::ICMP_EQ, "vceqz"); 4637651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgez_v: 4638651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgezq_v: 4639651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGE, 4640651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ICmpInst::ICMP_SGE, "vcgez"); 4641651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclez_v: 4642651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclezq_v: 4643651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLE, 4644651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ICmpInst::ICMP_SLE, "vclez"); 4645651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtz_v: 4646651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtzq_v: 4647651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT, 4648651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ICmpInst::ICMP_SGT, "vcgtz"); 4649651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltz_v: 4650651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltzq_v: 4651651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT, 4652651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ICmpInst::ICMP_SLT, "vcltz"); 4653651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_f64_v: 4654651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_f64_v: 465530d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4656651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float64, false, quad)); 4657258f930227c1a102c9c22eee88df65f748863425Jim Grosbach return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") 46589eb65a56e18bee1e5392bf2dff01cbd7b895f685Nate Begeman : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); 4659651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_f64_f32: { 4660651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines assert(Type.getEltType() == NeonTypeFlags::Float64 && quad && 4661651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "unexpected vcvt_f64_f32 builtin"); 4662651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float32, false, false); 4663651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); 4664651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4665651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFPExt(Ops[0], Ty, "vcvt"); 46669eb65a56e18bee1e5392bf2dff01cbd7b895f685Nate Begeman } 4667651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_f32_f64: { 4668651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines assert(Type.getEltType() == NeonTypeFlags::Float32 && 4669651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "unexpected vcvt_f32_f64 builtin"); 4670651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float64, false, true); 4671651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); 4672651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4673651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt"); 46749eb65a56e18bee1e5392bf2dff01cbd7b895f685Nate Begeman } 4675651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_s32_v: 4676651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_u32_v: 4677651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_s64_v: 4678651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_u64_v: 4679651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_s32_v: 4680651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_u32_v: 4681651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_s64_v: 4682651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_u64_v: { 4683651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Double = 4684651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 4685651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *InTy = 4686651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines GetNeonType(this, 4687651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags(Double ? NeonTypeFlags::Float64 4688651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : NeonTypeFlags::Float32, false, quad)); 4689651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], InTy); 4690651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (usgn) 4691651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFPToUI(Ops[0], Ty); 4692651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFPToSI(Ops[0], Ty); 4693651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4694651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvta_s32_v: 4695651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtaq_s32_v: 4696651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvta_u32_v: 4697651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtaq_u32_v: 4698651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvta_s64_v: 4699651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtaq_s64_v: 4700651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvta_u64_v: 4701651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtaq_u64_v: { 47026bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas; 4703651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Double = 4704651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 4705651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *InTy = 4706651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines GetNeonType(this, 4707651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags(Double ? NeonTypeFlags::Float64 4708651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : NeonTypeFlags::Float32, false, quad)); 4709651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, InTy }; 4710651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta"); 4711651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4712651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtm_s32_v: 4713651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtmq_s32_v: 4714651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtm_u32_v: 4715651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtmq_u32_v: 4716651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtm_s64_v: 4717651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtmq_s64_v: 4718651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtm_u64_v: 4719651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtmq_u64_v: { 47206bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms; 4721651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Double = 4722651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 4723651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *InTy = 4724651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines GetNeonType(this, 4725651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags(Double ? NeonTypeFlags::Float64 4726651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : NeonTypeFlags::Float32, false, quad)); 4727651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, InTy }; 4728651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm"); 4729651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4730651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtn_s32_v: 4731651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtnq_s32_v: 4732651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtn_u32_v: 4733651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtnq_u32_v: 4734651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtn_s64_v: 4735651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtnq_s64_v: 4736651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtn_u64_v: 4737651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtnq_u64_v: { 47386bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns; 4739651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Double = 4740651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 4741651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *InTy = 4742651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines GetNeonType(this, 4743651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags(Double ? NeonTypeFlags::Float64 4744651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : NeonTypeFlags::Float32, false, quad)); 4745651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, InTy }; 4746651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn"); 4747651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4748651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtp_s32_v: 4749651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtpq_s32_v: 4750651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtp_u32_v: 4751651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtpq_u32_v: 4752651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtp_s64_v: 4753651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtpq_s64_v: 4754651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtp_u64_v: 4755651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtpq_u64_v: { 47566bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps; 4757651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Double = 4758651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 4759651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *InTy = 4760651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines GetNeonType(this, 4761651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags(Double ? NeonTypeFlags::Float64 4762651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : NeonTypeFlags::Float32, false, quad)); 4763651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, InTy }; 4764651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp"); 4765651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4766651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmulx_v: 4767651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmulxq_v: { 47686bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_fmulx; 4769651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmulx"); 477030d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman } 4771651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmul_lane_v: 4772651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmul_laneq_v: { 4773651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // v1f64 vmul_lane should be mapped to Neon scalar mul lane 4774651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Quad = false; 4775651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v) 4776651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Quad = true; 4777651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 4778651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *VTy = GetNeonType(this, 4779651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags(NeonTypeFlags::Float64, false, Quad)); 4780651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], VTy); 4781651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract"); 4782651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Result = Builder.CreateFMul(Ops[0], Ops[1]); 4783651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateBitCast(Result, Ty); 4784651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4785651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vnegd_s64: 4786651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateNeg(EmitScalarExpr(E->getArg(0)), "vnegd"); 4787651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpmaxnm_v: 4788651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpmaxnmq_v: { 47896bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_fmaxnmp; 4790651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmaxnm"); 4791651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4792651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpminnm_v: 4793651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpminnmq_v: { 47946bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_fminnmp; 4795651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpminnm"); 4796651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4797651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsqrt_v: 4798651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsqrtq_v: { 4799651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Intrinsic::sqrt; 480030d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4801651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqrt"); 48021c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman } 4803651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrbit_v: 4804651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrbitq_v: { 48056bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_rbit; 4806651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrbit"); 4807651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4808651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddv_u8: 4809651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: These are handled by the AArch64 scalar code. 4810651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines usgn = true; 4811651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FALLTHROUGH 4812651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddv_s8: { 48136bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 4814651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 4815651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 4816651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); 4817651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 4818651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4819651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 4820651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 4821651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 8)); 4822651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4823651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddv_u16: 4824651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines usgn = true; 4825651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FALLTHROUGH 4826651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddv_s16: { 48276bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 4828651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 4829651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 4830651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); 4831651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 4832651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4833651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 4834651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 4835651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 4836651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4837651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddvq_u8: 4838651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines usgn = true; 4839651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FALLTHROUGH 4840651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddvq_s8: { 48416bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 4842651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 4843651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 4844651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); 4845651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 4846651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4847651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 4848651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 4849651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 8)); 4850651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4851651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddvq_u16: 4852651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines usgn = true; 4853651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FALLTHROUGH 4854651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddvq_s16: { 48556bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 4856651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 4857651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 4858651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); 4859651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 4860651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4861651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 4862651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 4863651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 4864651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4865651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxv_u8: { 48666bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_umaxv; 4867651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 4868651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 4869651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); 4870651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 4871651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4872651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 4873651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 4874651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 8)); 4875651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4876651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxv_u16: { 48776bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_umaxv; 4878651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 4879651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 4880651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); 4881651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 4882651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4883651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 4884651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 4885651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 4886651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4887651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxvq_u8: { 48886bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_umaxv; 4889651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 4890651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 4891651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); 4892651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 4893651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4894651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 4895651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 4896651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 8)); 4897651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4898651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxvq_u16: { 48996bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_umaxv; 4900651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 4901651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 4902651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); 4903651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 4904651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4905651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 4906651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 4907651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 4908651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4909651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxv_s8: { 49106bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_smaxv; 4911651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 4912651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 4913651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); 4914651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 4915651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4916651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 4917651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 4918651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 8)); 4919651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4920651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxv_s16: { 49216bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_smaxv; 4922651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 4923651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 4924651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); 4925651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 4926651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4927651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 4928651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 4929651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 4930651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4931651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxvq_s8: { 49326bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_smaxv; 4933651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 4934651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 4935651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); 4936651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 4937651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4938651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 4939651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 4940651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 8)); 4941651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4942651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxvq_s16: { 49436bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_smaxv; 4944651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 4945651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 4946651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); 4947651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 4948651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4949651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 4950651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 4951651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 4952651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4953651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminv_u8: { 49546bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_uminv; 4955651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 4956651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 4957651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); 4958651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 4959651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4960651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 4961651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 4962651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 8)); 4963651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4964651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminv_u16: { 49656bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_uminv; 4966651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 4967651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 4968651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); 4969651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 4970651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4971651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 4972651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 4973651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 4974651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4975651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminvq_u8: { 49766bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_uminv; 4977651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 4978651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 4979651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); 4980651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 4981651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4982651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 4983651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 4984651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 8)); 4985651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4986651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminvq_u16: { 49876bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_uminv; 4988651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 4989651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 4990651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); 4991651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 4992651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4993651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 4994651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 4995651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 4996651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4997651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminv_s8: { 49986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_sminv; 4999651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5000651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5001651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); 5002651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5003651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5004651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5005651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5006651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 8)); 5007651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5008651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminv_s16: { 50096bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_sminv; 5010651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5011651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5012651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); 5013651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5014651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5015651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5016651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5017651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 5018651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5019651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminvq_s8: { 50206bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_sminv; 5021651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5022651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5023651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); 5024651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5025651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5026651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5027651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5028651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 8)); 5029651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5030651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminvq_s16: { 50316bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_sminv; 5032651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5033651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5034651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); 5035651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5036651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5037651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5038651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5039651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 5040651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5041651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmul_n_f64: { 5042651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 5043651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *RHS = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), DoubleTy); 5044651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFMul(Ops[0], RHS); 5045651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5046651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddlv_u8: { 50476bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_uaddlv; 5048651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5049651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5050651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); 5051651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5052651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5053651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5054651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5055651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 5056651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5057651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddlv_u16: { 50586bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_uaddlv; 5059651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5060651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5061651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); 5062651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5063651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5064651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5065651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5066651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddlvq_u8: { 50676bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_uaddlv; 5068651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5069651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5070651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); 5071651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5072651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5073651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5074651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5075651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 5076651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5077651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddlvq_u16: { 50786bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_uaddlv; 5079651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5080651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5081651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); 5082651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5083651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5084651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5085651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5086651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddlv_s8: { 50876bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_saddlv; 5088651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5089651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5090651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); 5091651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5092651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5093651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5094651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5095651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 5096651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5097651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddlv_s16: { 50986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_saddlv; 5099651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5100651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5101651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); 5102651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5103651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5104651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5105651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5106651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddlvq_s8: { 51076bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_saddlv; 5108651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5109651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5110651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); 5111651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5112651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5113651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5114651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5115651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 5116651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5117651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddlvq_s16: { 51186bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_saddlv; 5119651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5120651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5121651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); 5122651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5123651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5124651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5125651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5126651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsri_n_v: 5127651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsriq_n_v: { 51286bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_vsri; 5129651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty); 5130651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(Intrin, Ops, "vsri_n"); 5131651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5132651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsli_n_v: 5133651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsliq_n_v: { 51346bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_vsli; 5135651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty); 5136651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(Intrin, Ops, "vsli_n"); 5137651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5138651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsra_n_v: 5139651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsraq_n_v: 5140651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5141651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); 5142651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Ops[0], Ops[1]); 5143651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrsra_n_v: 5144651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrsraq_n_v: { 51456bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl; 5146651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<llvm::Value*,2> TmpOps; 5147651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TmpOps.push_back(Ops[1]); 5148651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TmpOps.push_back(Ops[2]); 5149651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function* F = CGM.getIntrinsic(Int, Ty); 5150651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vrshr_n", 1, true); 5151651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], VTy); 5152651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Ops[0], tmp); 5153651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5154651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: Sharing loads & stores with 32-bit is complicated by the absence 5155651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // of an Align parameter here. 5156651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_x2_v: 5157651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_x2_v: 5158651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_x3_v: 5159651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_x3_v: 5160651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_x4_v: 5161651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_x4_v: { 5162651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType()); 5163651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5164651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, PTy }; 5165651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Int; 5166651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 5167651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_x2_v: 5168651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_x2_v: 51696bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_ld1x2; 5170651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 5171651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_x3_v: 5172651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_x3_v: 51736bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_ld1x3; 5174651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 5175651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_x4_v: 5176651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_x4_v: 51776bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_ld1x4; 5178651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 5179550a9d823a939366a9f776b58f18883acd905a93Bob Wilson } 5180651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(Int, Tys); 5181651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN"); 5182651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 5183651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5184651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateStore(Ops[1], Ops[0]); 5185651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5186651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_x2_v: 5187651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_x2_v: 5188651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_x3_v: 5189651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_x3_v: 5190651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_x4_v: 5191651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_x4_v: { 5192651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType()); 5193651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, PTy }; 5194651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Int; 5195651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 5196651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_x2_v: 5197651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_x2_v: 51986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_st1x2; 5199651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 5200651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_x3_v: 5201651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_x3_v: 52026bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_st1x3; 5203651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 5204651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_x4_v: 5205651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_x4_v: 52066bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_st1x4; 5207651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 5208651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5209651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value *, 4> IntOps(Ops.begin()+1, Ops.end()); 5210651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines IntOps.push_back(Ops[0]); 5211651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), IntOps, ""); 5212651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5213651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_v: 5214651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_v: 5215651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); 5216651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateLoad(Ops[0]); 5217651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_v: 5218651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_v: 5219651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); 5220651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], VTy); 5221651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateStore(Ops[1], Ops[0]); 5222651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_lane_v: 5223651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_lane_v: 52244be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 52254be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ty = llvm::PointerType::getUnqual(VTy->getElementType()); 52264be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5227651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateLoad(Ops[0]); 5228651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane"); 5229651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_dup_v: 5230651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_dup_v: { 52314be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Value *V = UndefValue::get(Ty); 52324be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ty = llvm::PointerType::getUnqual(VTy->getElementType()); 52334be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5234651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateLoad(Ops[0]); 523577b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner llvm::Constant *CI = ConstantInt::get(Int32Ty, 0); 5236651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI); 52374be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman return EmitNeonSplat(Ops[0], CI); 52384be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 5239651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_lane_v: 5240651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_lane_v: 5241651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 5242651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); 52434be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 5244651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateStore(Ops[1], Builder.CreateBitCast(Ops[0], Ty)); 5245651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_v: 5246651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2q_v: { 5247651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *PTy = llvm::PointerType::getUnqual(VTy); 5248651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5249651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, PTy }; 52506bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2, Tys); 5251651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); 5252651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 5253651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(Ops[1]->getType())); 52544be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman return Builder.CreateStore(Ops[1], Ops[0]); 52554be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 5256651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_v: 5257651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3q_v: { 5258651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *PTy = llvm::PointerType::getUnqual(VTy); 5259651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5260651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, PTy }; 52616bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3, Tys); 5262651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); 5263651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 5264651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(Ops[1]->getType())); 52654be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman return Builder.CreateStore(Ops[1], Ops[0]); 52664be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 5267651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_v: 5268651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4q_v: { 5269651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *PTy = llvm::PointerType::getUnqual(VTy); 5270651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5271651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, PTy }; 52726bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4, Tys); 5273651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); 5274651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 5275651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(Ops[1]->getType())); 5276651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateStore(Ops[1], Ops[0]); 5277651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5278651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_dup_v: 5279651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2q_dup_v: { 5280651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *PTy = 5281651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(VTy->getElementType()); 5282651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5283651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, PTy }; 52846bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2r, Tys); 5285651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); 5286651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 5287651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(Ops[1]->getType())); 52884be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman return Builder.CreateStore(Ops[1], Ops[0]); 52894be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 5290651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_dup_v: 5291651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3q_dup_v: { 5292651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *PTy = 5293651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(VTy->getElementType()); 5294651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5295651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, PTy }; 52966bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3r, Tys); 5297651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); 5298651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 5299651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(Ops[1]->getType())); 5300651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateStore(Ops[1], Ops[0]); 5301651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5302651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_dup_v: 5303651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4q_dup_v: { 5304651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *PTy = 5305651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(VTy->getElementType()); 5306651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5307651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, PTy }; 53086bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4r, Tys); 5309651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); 5310651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 5311651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(Ops[1]->getType())); 5312651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateStore(Ops[1], Ops[0]); 5313651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5314651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_lane_v: 5315651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2q_lane_v: { 5316651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; 53176bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2lane, Tys); 5318651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[1]); 5319651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()+1); 5320651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 53214be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 5322651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[3] = Builder.CreateZExt(Ops[3], 5323651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 64)); 5324651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, 5325651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ArrayRef<Value*>(Ops).slice(1), "vld2_lane"); 53264be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 53274be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 53284be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman return Builder.CreateStore(Ops[1], Ops[0]); 53294be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 5330651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_lane_v: 5331651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3q_lane_v: { 5332651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; 53336bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3lane, Tys); 5334651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[1]); 5335651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()+1); 5336651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 53374be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 53384be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[3] = Builder.CreateBitCast(Ops[3], Ty); 5339651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[4] = Builder.CreateZExt(Ops[4], 5340651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 64)); 5341651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, 5342651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ArrayRef<Value*>(Ops).slice(1), "vld3_lane"); 53434be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 53444be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 53454be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman return Builder.CreateStore(Ops[1], Ops[0]); 53464be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 5347651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_lane_v: 5348651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4q_lane_v: { 5349651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; 53506bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4lane, Tys); 5351651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[1]); 5352651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()+1); 5353651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 53544be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 53554be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[3] = Builder.CreateBitCast(Ops[3], Ty); 53564be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[4] = Builder.CreateBitCast(Ops[4], Ty); 5357651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[5] = Builder.CreateZExt(Ops[5], 5358651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 64)); 5359651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, 5360651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ArrayRef<Value*>(Ops).slice(1), "vld4_lane"); 53614be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 53624be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 53634be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman return Builder.CreateStore(Ops[1], Ops[0]); 53644be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 5365651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2_v: 5366651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2q_v: { 5367651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[0]); 5368651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()); 5369651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[2]->getType() }; 53706bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2, Tys), 5371464ccb68f22a7e1c0a2844551c16f721540c91c3Nate Begeman Ops, ""); 5372eac1f6746a2915fea3ed42284b07e274c0095a95Bob Wilson } 5373651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2_lane_v: 5374651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2q_lane_v: { 5375651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[0]); 5376651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()); 5377651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateZExt(Ops[2], 5378651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 64)); 5379651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; 53806bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2lane, Tys), 5381464ccb68f22a7e1c0a2844551c16f721540c91c3Nate Begeman Ops, ""); 5382651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5383651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3_v: 5384651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3q_v: { 5385651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[0]); 5386651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()); 5387651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; 53886bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3, Tys), 5389464ccb68f22a7e1c0a2844551c16f721540c91c3Nate Begeman Ops, ""); 5390651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5391651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3_lane_v: 5392651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3q_lane_v: { 5393651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[0]); 5394651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()); 5395651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[3] = Builder.CreateZExt(Ops[3], 5396651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 64)); 5397651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; 53986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3lane, Tys), 5399464ccb68f22a7e1c0a2844551c16f721540c91c3Nate Begeman Ops, ""); 5400651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5401651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4_v: 5402651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4q_v: { 5403651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[0]); 5404651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()); 5405651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; 54066bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4, Tys), 5407464ccb68f22a7e1c0a2844551c16f721540c91c3Nate Begeman Ops, ""); 54086bf1e8eadd36f077cf5403d23c664129d98eaf79Tim Northover } 5409651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4_lane_v: 5410651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4q_lane_v: { 5411651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[0]); 5412651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()); 5413651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[4] = Builder.CreateZExt(Ops[4], 5414651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 64)); 5415651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[5]->getType() }; 54166bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4lane, Tys), 5417651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, ""); 54181c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman } 5419651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtrn_v: 5420651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtrnq_v: { 54214be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 54224be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 54234be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 54246bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *SV = nullptr; 54254be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman 54261c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman for (unsigned vi = 0; vi != 2; ++vi) { 54274be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman SmallVector<Constant*, 16> Indices; 54284be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 5429651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Indices.push_back(ConstantInt::get(Int32Ty, i+vi)); 5430651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Indices.push_back(ConstantInt::get(Int32Ty, i+e+vi)); 54311c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman } 54324be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); 5433fb018d153cbf12fd2a4a278cbf7614b9a2e2835eChris Lattner SV = llvm::ConstantVector::get(Indices); 54344be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn"); 54354be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman SV = Builder.CreateStore(SV, Addr); 54361c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman } 54374be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman return SV; 54381c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman } 5439651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vuzp_v: 5440651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vuzpq_v: { 54414be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 54421c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 54434be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 54446bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *SV = nullptr; 5445258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 54464be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman for (unsigned vi = 0; vi != 2; ++vi) { 54474be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman SmallVector<Constant*, 16> Indices; 54484be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) 544977b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner Indices.push_back(ConstantInt::get(Int32Ty, 2*i+vi)); 54504be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman 54514be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); 5452fb018d153cbf12fd2a4a278cbf7614b9a2e2835eChris Lattner SV = llvm::ConstantVector::get(Indices); 54534be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp"); 54544be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman SV = Builder.CreateStore(SV, Addr); 54554be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 54564be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman return SV; 54571c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman } 5458651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vzip_v: 5459651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vzipq_v: { 54604be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 54611c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 54624be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 54636bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *SV = nullptr; 5464258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 54654be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman for (unsigned vi = 0; vi != 2; ++vi) { 54664be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman SmallVector<Constant*, 16> Indices; 54674be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 5468e361cc3daa6c22e4413d48bd8b8934ea9fd5a55fDaniel Dunbar Indices.push_back(ConstantInt::get(Int32Ty, (i + vi*e) >> 1)); 5469e361cc3daa6c22e4413d48bd8b8934ea9fd5a55fDaniel Dunbar Indices.push_back(ConstantInt::get(Int32Ty, ((i + vi*e) >> 1)+e)); 54704be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 54714be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); 5472fb018d153cbf12fd2a4a278cbf7614b9a2e2835eChris Lattner SV = llvm::ConstantVector::get(Indices); 54734be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip"); 54744be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman SV = Builder.CreateStore(SV, Addr); 54754be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 54764be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman return SV; 54779eb65a56e18bee1e5392bf2dff01cbd7b895f685Nate Begeman } 5478651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl1q_v: { 54796bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl1, Ty), 5480651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbl1"); 5481651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5482651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl2q_v: { 54836bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl2, Ty), 5484651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbl2"); 5485651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5486651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl3q_v: { 54876bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl3, Ty), 5488651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbl3"); 5489651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5490651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl4q_v: { 54916bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl4, Ty), 5492651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbl4"); 5493651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5494651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx1q_v: { 54956bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx1, Ty), 5496651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbx1"); 5497651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5498651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx2q_v: { 54996bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx2, Ty), 5500651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbx2"); 5501651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5502651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx3q_v: { 55036bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx3, Ty), 5504651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbx3"); 5505651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5506651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx4q_v: { 55076bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx4, Ty), 5508651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbx4"); 5509651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5510651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsqadd_v: 5511651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsqaddq_v: { 55126bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_usqadd; 5513651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqadd"); 5514651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5515651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vuqadd_v: 5516651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vuqaddq_v: { 55176bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_suqadd; 5518651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vuqadd"); 5519651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 55202752c0137d95aa2f4ee1cdff4b564bac842e041bChris Lattner } 55212752c0137d95aa2f4ee1cdff4b564bac842e041bChris Lattner} 55222752c0137d95aa2f4ee1cdff4b564bac842e041bChris Lattner 5523aa51e513850688b7963efc62abf1eface7037602Bill Wendlingllvm::Value *CodeGenFunction:: 5524795b10062c2eaffae9e04241fb1a73cdbcb24a37Bill WendlingBuildVector(ArrayRef<llvm::Value*> Ops) { 5525aa51e513850688b7963efc62abf1eface7037602Bill Wendling assert((Ops.size() & (Ops.size() - 1)) == 0 && 5526aa51e513850688b7963efc62abf1eface7037602Bill Wendling "Not a power-of-two sized vector!"); 5527aa51e513850688b7963efc62abf1eface7037602Bill Wendling bool AllConstants = true; 5528aa51e513850688b7963efc62abf1eface7037602Bill Wendling for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i) 5529aa51e513850688b7963efc62abf1eface7037602Bill Wendling AllConstants &= isa<Constant>(Ops[i]); 5530aa51e513850688b7963efc62abf1eface7037602Bill Wendling 5531aa51e513850688b7963efc62abf1eface7037602Bill Wendling // If this is a constant vector, create a ConstantVector. 5532aa51e513850688b7963efc62abf1eface7037602Bill Wendling if (AllConstants) { 55332ce8842641cc312628c4be836d34eb250e7c3f78Chris Lattner SmallVector<llvm::Constant*, 16> CstOps; 5534aa51e513850688b7963efc62abf1eface7037602Bill Wendling for (unsigned i = 0, e = Ops.size(); i != e; ++i) 5535aa51e513850688b7963efc62abf1eface7037602Bill Wendling CstOps.push_back(cast<Constant>(Ops[i])); 5536aa51e513850688b7963efc62abf1eface7037602Bill Wendling return llvm::ConstantVector::get(CstOps); 5537aa51e513850688b7963efc62abf1eface7037602Bill Wendling } 5538aa51e513850688b7963efc62abf1eface7037602Bill Wendling 5539aa51e513850688b7963efc62abf1eface7037602Bill Wendling // Otherwise, insertelement the values to build the vector. 5540aa51e513850688b7963efc62abf1eface7037602Bill Wendling Value *Result = 5541aa51e513850688b7963efc62abf1eface7037602Bill Wendling llvm::UndefValue::get(llvm::VectorType::get(Ops[0]->getType(), Ops.size())); 5542aa51e513850688b7963efc62abf1eface7037602Bill Wendling 5543aa51e513850688b7963efc62abf1eface7037602Bill Wendling for (unsigned i = 0, e = Ops.size(); i != e; ++i) 55442ce8842641cc312628c4be836d34eb250e7c3f78Chris Lattner Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt32(i)); 5545aa51e513850688b7963efc62abf1eface7037602Bill Wendling 5546aa51e513850688b7963efc62abf1eface7037602Bill Wendling return Result; 5547aa51e513850688b7963efc62abf1eface7037602Bill Wendling} 5548aa51e513850688b7963efc62abf1eface7037602Bill Wendling 55491eb4433ac451dc16f4133a88af2d002ac26c58efMike StumpValue *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, 55501feedd84221e8dbcc3faf3de27cc42b559db845dChris Lattner const CallExpr *E) { 55515f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner SmallVector<Value*, 4> Ops; 55522929cfa9b7df1d5b0571b54161783e4d791a0b77Anders Carlsson 555346c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner // Find out if any arguments are required to be integer constant expressions. 555446c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner unsigned ICEArguments = 0; 555546c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner ASTContext::GetBuiltinTypeError Error; 555646c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); 555746c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner assert(Error == ASTContext::GE_None && "Should not codegen an error"); 555846c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner 555946c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) { 556046c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner // If this is a normal argument, just emit it as a scalar. 556146c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner if ((ICEArguments & (1 << i)) == 0) { 556246c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner Ops.push_back(EmitScalarExpr(E->getArg(i))); 556346c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner continue; 556446c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner } 556546c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner 556646c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner // If this is required to be a constant, constant fold it so that we know 556746c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner // that the generated intrinsic gets a ConstantInt. 556846c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner llvm::APSInt Result; 556946c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext()); 557046c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst; 5571d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); 557246c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner } 55732929cfa9b7df1d5b0571b54161783e4d791a0b77Anders Carlsson 5574564f1de67d7ba43646b8740db86d6269e3dfbe0bAnders Carlsson switch (BuiltinID) { 55756bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines default: return nullptr; 5576651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case X86::BI_mm_prefetch: { 5577651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Address = EmitScalarExpr(E->getArg(0)); 5578651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *RW = ConstantInt::get(Int32Ty, 0); 5579651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Locality = EmitScalarExpr(E->getArg(1)); 5580651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Data = ConstantInt::get(Int32Ty, 1); 5581651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *F = CGM.getIntrinsic(Intrinsic::prefetch); 5582651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall4(F, Address, RW, Locality, Data); 5583651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5584aa51e513850688b7963efc62abf1eface7037602Bill Wendling case X86::BI__builtin_ia32_vec_init_v8qi: 5585aa51e513850688b7963efc62abf1eface7037602Bill Wendling case X86::BI__builtin_ia32_vec_init_v4hi: 5586aa51e513850688b7963efc62abf1eface7037602Bill Wendling case X86::BI__builtin_ia32_vec_init_v2si: 5587aa51e513850688b7963efc62abf1eface7037602Bill Wendling return Builder.CreateBitCast(BuildVector(Ops), 5588d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall llvm::Type::getX86_MMXTy(getLLVMContext())); 55891944ec188408aff1931c62c79a069e30f2549ec2Argyrios Kyrtzidis case X86::BI__builtin_ia32_vec_ext_v2si: 55901944ec188408aff1931c62c79a069e30f2549ec2Argyrios Kyrtzidis return Builder.CreateExtractElement(Ops[0], 55911944ec188408aff1931c62c79a069e30f2549ec2Argyrios Kyrtzidis llvm::ConstantInt::get(Ops[1]->getType(), 0)); 5592e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman case X86::BI__builtin_ia32_ldmxcsr: { 55933fc779191ce87cddf603e0c5bdbd225611201ff4Juergen Ributzka Value *Tmp = CreateMemTemp(E->getArg(0)->getType()); 5594e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman Builder.CreateStore(Ops[0], Tmp); 5595e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr), 55963fc779191ce87cddf603e0c5bdbd225611201ff4Juergen Ributzka Builder.CreateBitCast(Tmp, Int8PtrTy)); 5597e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman } 5598e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman case X86::BI__builtin_ia32_stmxcsr: { 5599fe0af454943374758309a1066e3304b3a56af04eJuergen Ributzka Value *Tmp = CreateMemTemp(E->getType()); 5600012614ecf78442368ec82ee30efb3bc047b413e6Ted Kremenek Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr), 56013fc779191ce87cddf603e0c5bdbd225611201ff4Juergen Ributzka Builder.CreateBitCast(Tmp, Int8PtrTy)); 5602e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman return Builder.CreateLoad(Tmp, "stmxcsr"); 5603e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman } 5604e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman case X86::BI__builtin_ia32_storehps: 5605e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman case X86::BI__builtin_ia32_storelps: { 560677b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner llvm::Type *PtrTy = llvm::PointerType::getUnqual(Int64Ty); 560777b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2); 56081eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 5609e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman // cast val v2i64 5610e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman Ops[1] = Builder.CreateBitCast(Ops[1], VecTy, "cast"); 56111eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 5612e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman // extract (0, 1) 5613e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman unsigned Index = BuiltinID == X86::BI__builtin_ia32_storelps ? 0 : 1; 561477b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner llvm::Value *Idx = llvm::ConstantInt::get(Int32Ty, Index); 5615e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman Ops[1] = Builder.CreateExtractElement(Ops[1], Idx, "extract"); 5616e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman 5617e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman // cast pointer to i64 & store 5618e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], PtrTy); 5619e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman return Builder.CreateStore(Ops[1], Ops[0]); 5620e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman } 562128cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling case X86::BI__builtin_ia32_palignr: { 562228cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); 5623258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 562428cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling // If palignr is shifting the pair of input vectors less than 9 bytes, 562528cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling // emit a shuffle instruction. 562628cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling if (shiftVal <= 8) { 56275f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner SmallVector<llvm::Constant*, 8> Indices; 562828cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling for (unsigned i = 0; i != 8; ++i) 562928cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling Indices.push_back(llvm::ConstantInt::get(Int32Ty, shiftVal + i)); 5630258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 5631fb018d153cbf12fd2a4a278cbf7614b9a2e2835eChris Lattner Value* SV = llvm::ConstantVector::get(Indices); 563228cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr"); 563328cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling } 5634258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 563528cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling // If palignr is shifting the pair of input vectors more than 8 but less 563628cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling // than 16 bytes, emit a logical right shift of the destination. 563728cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling if (shiftVal < 16) { 563828cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling // MMX has these as 1 x i64 vectors for some odd optimization reasons. 56392acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 1); 5640258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 564128cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); 564228cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling Ops[1] = llvm::ConstantInt::get(VecTy, (shiftVal-8) * 8); 5643258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 564428cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling // create i32 constant 564528cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_mmx_psrl_q); 5646e9c0265d6e6b5bf865f4a0c2c00d00ac251e6437Frits van Bommel return Builder.CreateCall(F, makeArrayRef(&Ops[0], 2), "palignr"); 564728cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling } 5648258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 56495c22ad2ef6bf39da22d5190025e0ddfd4b568b2aEli Friedman // If palignr is shifting the pair of vectors more than 16 bytes, emit zero. 565028cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling return llvm::Constant::getNullValue(ConvertType(E->getType())); 565128cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling } 5652c3420ffb282c6ffc0192013bf8045b6c21eddeceNate Begeman case X86::BI__builtin_ia32_palignr128: { 5653ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); 5654258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 5655ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman // If palignr is shifting the pair of input vectors less than 17 bytes, 5656ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman // emit a shuffle instruction. 5657ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman if (shiftVal <= 16) { 56585f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner SmallVector<llvm::Constant*, 16> Indices; 5659ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman for (unsigned i = 0; i != 16; ++i) 566077b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner Indices.push_back(llvm::ConstantInt::get(Int32Ty, shiftVal + i)); 5661258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 5662fb018d153cbf12fd2a4a278cbf7614b9a2e2835eChris Lattner Value* SV = llvm::ConstantVector::get(Indices); 5663ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr"); 5664ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman } 5665258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 5666ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman // If palignr is shifting the pair of input vectors more than 16 but less 5667ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman // than 32 bytes, emit a logical right shift of the destination. 5668ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman if (shiftVal < 32) { 56692acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2); 5670258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 5671ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); 567277b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner Ops[1] = llvm::ConstantInt::get(Int32Ty, (shiftVal-16) * 8); 5673258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 5674ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman // create i32 constant 5675ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_sse2_psrl_dq); 5676e9c0265d6e6b5bf865f4a0c2c00d00ac251e6437Frits van Bommel return Builder.CreateCall(F, makeArrayRef(&Ops[0], 2), "palignr"); 5677ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman } 5678258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 5679ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman // If palignr is shifting the pair of vectors more than 32 bytes, emit zero. 5680ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman return llvm::Constant::getNullValue(ConvertType(E->getType())); 568191b59274439f776cdd545bd7bf5849fdb1842160Eric Christopher } 56829c2ffd803af03f1728423d0d73ff87d988642633Craig Topper case X86::BI__builtin_ia32_palignr256: { 56839c2ffd803af03f1728423d0d73ff87d988642633Craig Topper unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); 56849c2ffd803af03f1728423d0d73ff87d988642633Craig Topper 56859c2ffd803af03f1728423d0d73ff87d988642633Craig Topper // If palignr is shifting the pair of input vectors less than 17 bytes, 56869c2ffd803af03f1728423d0d73ff87d988642633Craig Topper // emit a shuffle instruction. 56879c2ffd803af03f1728423d0d73ff87d988642633Craig Topper if (shiftVal <= 16) { 56889c2ffd803af03f1728423d0d73ff87d988642633Craig Topper SmallVector<llvm::Constant*, 32> Indices; 56899c2ffd803af03f1728423d0d73ff87d988642633Craig Topper // 256-bit palignr operates on 128-bit lanes so we need to handle that 56909c2ffd803af03f1728423d0d73ff87d988642633Craig Topper for (unsigned l = 0; l != 2; ++l) { 56919c2ffd803af03f1728423d0d73ff87d988642633Craig Topper unsigned LaneStart = l * 16; 56929c2ffd803af03f1728423d0d73ff87d988642633Craig Topper unsigned LaneEnd = (l+1) * 16; 56939c2ffd803af03f1728423d0d73ff87d988642633Craig Topper for (unsigned i = 0; i != 16; ++i) { 56949c2ffd803af03f1728423d0d73ff87d988642633Craig Topper unsigned Idx = shiftVal + i + LaneStart; 56959c2ffd803af03f1728423d0d73ff87d988642633Craig Topper if (Idx >= LaneEnd) Idx += 16; // end of lane, switch operand 56969c2ffd803af03f1728423d0d73ff87d988642633Craig Topper Indices.push_back(llvm::ConstantInt::get(Int32Ty, Idx)); 56979c2ffd803af03f1728423d0d73ff87d988642633Craig Topper } 56989c2ffd803af03f1728423d0d73ff87d988642633Craig Topper } 56999c2ffd803af03f1728423d0d73ff87d988642633Craig Topper 57009c2ffd803af03f1728423d0d73ff87d988642633Craig Topper Value* SV = llvm::ConstantVector::get(Indices); 57019c2ffd803af03f1728423d0d73ff87d988642633Craig Topper return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr"); 57029c2ffd803af03f1728423d0d73ff87d988642633Craig Topper } 57039c2ffd803af03f1728423d0d73ff87d988642633Craig Topper 57049c2ffd803af03f1728423d0d73ff87d988642633Craig Topper // If palignr is shifting the pair of input vectors more than 16 but less 57059c2ffd803af03f1728423d0d73ff87d988642633Craig Topper // than 32 bytes, emit a logical right shift of the destination. 57069c2ffd803af03f1728423d0d73ff87d988642633Craig Topper if (shiftVal < 32) { 57079c2ffd803af03f1728423d0d73ff87d988642633Craig Topper llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 4); 57089c2ffd803af03f1728423d0d73ff87d988642633Craig Topper 57099c2ffd803af03f1728423d0d73ff87d988642633Craig Topper Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); 57109c2ffd803af03f1728423d0d73ff87d988642633Craig Topper Ops[1] = llvm::ConstantInt::get(Int32Ty, (shiftVal-16) * 8); 57119c2ffd803af03f1728423d0d73ff87d988642633Craig Topper 57129c2ffd803af03f1728423d0d73ff87d988642633Craig Topper // create i32 constant 57139c2ffd803af03f1728423d0d73ff87d988642633Craig Topper llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_avx2_psrl_dq); 57149c2ffd803af03f1728423d0d73ff87d988642633Craig Topper return Builder.CreateCall(F, makeArrayRef(&Ops[0], 2), "palignr"); 57159c2ffd803af03f1728423d0d73ff87d988642633Craig Topper } 57169c2ffd803af03f1728423d0d73ff87d988642633Craig Topper 57179c2ffd803af03f1728423d0d73ff87d988642633Craig Topper // If palignr is shifting the pair of vectors more than 32 bytes, emit zero. 57189c2ffd803af03f1728423d0d73ff87d988642633Craig Topper return llvm::Constant::getNullValue(ConvertType(E->getType())); 57199c2ffd803af03f1728423d0d73ff87d988642633Craig Topper } 5720b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling case X86::BI__builtin_ia32_movntps: 57214a7376d00df98cbfcad29ee709637707a2f3d46cCraig Topper case X86::BI__builtin_ia32_movntps256: 5722b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling case X86::BI__builtin_ia32_movntpd: 57234a7376d00df98cbfcad29ee709637707a2f3d46cCraig Topper case X86::BI__builtin_ia32_movntpd256: 5724b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling case X86::BI__builtin_ia32_movntdq: 57254a7376d00df98cbfcad29ee709637707a2f3d46cCraig Topper case X86::BI__builtin_ia32_movntdq256: 5726440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman case X86::BI__builtin_ia32_movnti: 5727440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman case X86::BI__builtin_ia32_movnti64: { 5728b107dd02f5e365f60b036b8a27c784f2d0d0a855Bill Wendling llvm::MDNode *Node = llvm::MDNode::get(getLLVMContext(), 5729b107dd02f5e365f60b036b8a27c784f2d0d0a855Bill Wendling Builder.getInt32(1)); 5730b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling 5731b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling // Convert the type of the pointer to a pointer to the stored type. 5732b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling Value *BC = Builder.CreateBitCast(Ops[0], 5733b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling llvm::PointerType::getUnqual(Ops[1]->getType()), 5734b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling "cast"); 5735b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling StoreInst *SI = Builder.CreateStore(Ops[1], BC); 5736b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling SI->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node); 5737440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman 5738440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman // If the operand is an integer, we can't assume alignment. Otherwise, 5739440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman // assume natural alignment. 5740440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman QualType ArgTy = E->getArg(1)->getType(); 5741440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman unsigned Align; 5742440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman if (ArgTy->isIntegerType()) 5743440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman Align = 1; 5744440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman else 5745440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman Align = getContext().getTypeSizeInChars(ArgTy).getQuantity(); 5746440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman SI->setAlignment(Align); 5747b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling return SI; 5748b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling } 57498b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer // 3DNow! 57508b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer case X86::BI__builtin_ia32_pswapdsf: 57518b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer case X86::BI__builtin_ia32_pswapdsi: { 57526bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines const char *name = nullptr; 57538b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer Intrinsic::ID ID = Intrinsic::not_intrinsic; 57548b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer switch(BuiltinID) { 5755f8495d67ca4dd2ea15a4dc59e9a2fa32a9bfa475Craig Topper default: llvm_unreachable("Unsupported intrinsic!"); 57568b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer case X86::BI__builtin_ia32_pswapdsf: 57578b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer case X86::BI__builtin_ia32_pswapdsi: 57588b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer name = "pswapd"; 57598b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer ID = Intrinsic::x86_3dnowa_pswapd; 57608b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer break; 57618b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer } 5762345032a7211a6f983d59c30c0b3fa2b96819532aChandler Carruth llvm::Type *MMXTy = llvm::Type::getX86_MMXTy(getLLVMContext()); 5763345032a7211a6f983d59c30c0b3fa2b96819532aChandler Carruth Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy, "cast"); 57648b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer llvm::Function *F = CGM.getIntrinsic(ID); 57654c7d9f1507d0f102bd4133bba63348636facd469Jay Foad return Builder.CreateCall(F, Ops, name); 57668b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer } 57679a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer case X86::BI__builtin_ia32_rdrand16_step: 57689a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer case X86::BI__builtin_ia32_rdrand32_step: 57691bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao case X86::BI__builtin_ia32_rdrand64_step: 57701bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao case X86::BI__builtin_ia32_rdseed16_step: 57711bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao case X86::BI__builtin_ia32_rdseed32_step: 57721bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao case X86::BI__builtin_ia32_rdseed64_step: { 57739a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer Intrinsic::ID ID; 57749a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer switch (BuiltinID) { 57759a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer default: llvm_unreachable("Unsupported intrinsic!"); 57769a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer case X86::BI__builtin_ia32_rdrand16_step: 57779a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer ID = Intrinsic::x86_rdrand_16; 57789a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer break; 57799a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer case X86::BI__builtin_ia32_rdrand32_step: 57809a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer ID = Intrinsic::x86_rdrand_32; 57819a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer break; 57829a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer case X86::BI__builtin_ia32_rdrand64_step: 57839a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer ID = Intrinsic::x86_rdrand_64; 57849a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer break; 57851bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao case X86::BI__builtin_ia32_rdseed16_step: 57861bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao ID = Intrinsic::x86_rdseed_16; 57871bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao break; 57881bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao case X86::BI__builtin_ia32_rdseed32_step: 57891bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao ID = Intrinsic::x86_rdseed_32; 57901bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao break; 57911bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao case X86::BI__builtin_ia32_rdseed64_step: 57921bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao ID = Intrinsic::x86_rdseed_64; 57931bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao break; 57949a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer } 57959a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer 57969a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID)); 57979a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer Builder.CreateStore(Builder.CreateExtractValue(Call, 0), Ops[0]); 57989a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer return Builder.CreateExtractValue(Call, 1); 57999a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer } 58002766deb114cc5d3420027764438cf683dda8a9f0Juergen Ributzka // AVX2 broadcast 58012766deb114cc5d3420027764438cf683dda8a9f0Juergen Ributzka case X86::BI__builtin_ia32_vbroadcastsi256: { 58023fc779191ce87cddf603e0c5bdbd225611201ff4Juergen Ributzka Value *VecTmp = CreateMemTemp(E->getArg(0)->getType()); 58033fc779191ce87cddf603e0c5bdbd225611201ff4Juergen Ributzka Builder.CreateStore(Ops[0], VecTmp); 58043fc779191ce87cddf603e0c5bdbd225611201ff4Juergen Ributzka Value *F = CGM.getIntrinsic(Intrinsic::x86_avx2_vbroadcasti128); 58053fc779191ce87cddf603e0c5bdbd225611201ff4Juergen Ributzka return Builder.CreateCall(F, Builder.CreateBitCast(VecTmp, Int8PtrTy)); 58062766deb114cc5d3420027764438cf683dda8a9f0Juergen Ributzka } 5807564f1de67d7ba43646b8740db86d6269e3dfbe0bAnders Carlsson } 5808564f1de67d7ba43646b8740db86d6269e3dfbe0bAnders Carlsson} 5809564f1de67d7ba43646b8740db86d6269e3dfbe0bAnders Carlsson 58109631939f82c0eaa6fb3936a0ce58a41adfbc9011Tony Linthicum 58111eb4433ac451dc16f4133a88af2d002ac26c58efMike StumpValue *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, 58121feedd84221e8dbcc3faf3de27cc42b559db845dChris Lattner const CallExpr *E) { 58135f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner SmallVector<Value*, 4> Ops; 5814dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner 5815dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) 5816dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner Ops.push_back(EmitScalarExpr(E->getArg(i))); 5817dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner 5818dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner Intrinsic::ID ID = Intrinsic::not_intrinsic; 5819dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner 5820dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner switch (BuiltinID) { 58216bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines default: return nullptr; 5822dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner 58234d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov // vec_ld, vec_lvsl, vec_lvsr 58244d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvx: 58254d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvxl: 58264d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvebx: 58274d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvehx: 58284d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvewx: 58294d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvsl: 58304d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvsr: 58314d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov { 5832d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy); 58334d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov 5834578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer Ops[0] = Builder.CreateGEP(Ops[1], Ops[0]); 58354d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov Ops.pop_back(); 58364d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov 58374d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov switch (BuiltinID) { 5838b219cfc4d75f0a03630b7c4509ef791b7e97b2c8David Blaikie default: llvm_unreachable("Unsupported ld/lvsl/lvsr intrinsic!"); 58394d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvx: 58404d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov ID = Intrinsic::ppc_altivec_lvx; 58414d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov break; 58424d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvxl: 58434d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov ID = Intrinsic::ppc_altivec_lvxl; 58444d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov break; 58454d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvebx: 58464d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov ID = Intrinsic::ppc_altivec_lvebx; 58474d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov break; 58484d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvehx: 58494d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov ID = Intrinsic::ppc_altivec_lvehx; 58504d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov break; 58514d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvewx: 58524d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov ID = Intrinsic::ppc_altivec_lvewx; 58534d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov break; 58544d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvsl: 58554d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov ID = Intrinsic::ppc_altivec_lvsl; 58564d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov break; 58574d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvsr: 58584d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov ID = Intrinsic::ppc_altivec_lvsr; 58594d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov break; 58604d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov } 58614d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov llvm::Function *F = CGM.getIntrinsic(ID); 58624c7d9f1507d0f102bd4133bba63348636facd469Jay Foad return Builder.CreateCall(F, Ops, ""); 58634d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov } 58644d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov 5865dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner // vec_st 5866dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvx: 5867dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvxl: 5868dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvebx: 5869dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvehx: 5870dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvewx: 5871dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner { 5872d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy); 5873578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer Ops[1] = Builder.CreateGEP(Ops[2], Ops[1]); 5874dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner Ops.pop_back(); 5875dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner 5876dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner switch (BuiltinID) { 5877b219cfc4d75f0a03630b7c4509ef791b7e97b2c8David Blaikie default: llvm_unreachable("Unsupported st intrinsic!"); 5878dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvx: 5879dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner ID = Intrinsic::ppc_altivec_stvx; 5880dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner break; 5881dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvxl: 5882dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner ID = Intrinsic::ppc_altivec_stvxl; 5883dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner break; 5884dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvebx: 5885dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner ID = Intrinsic::ppc_altivec_stvebx; 5886dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner break; 5887dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvehx: 5888dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner ID = Intrinsic::ppc_altivec_stvehx; 5889dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner break; 5890dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvewx: 5891dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner ID = Intrinsic::ppc_altivec_stvewx; 5892dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner break; 5893dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner } 5894dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner llvm::Function *F = CGM.getIntrinsic(ID); 58954c7d9f1507d0f102bd4133bba63348636facd469Jay Foad return Builder.CreateCall(F, Ops, ""); 5896dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner } 5897dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner } 58981eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump} 5899