CGBuiltin.cpp revision 176edba5311f6eff0cad2631449885ddf4fbc9ea
1022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson//===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===// 2022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson// 3022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson// The LLVM Compiler Infrastructure 4022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson// 50bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// This file is distributed under the University of Illinois Open Source 60bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// License. See LICENSE.TXT for details. 7022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson// 8022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson//===----------------------------------------------------------------------===// 9022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson// 10022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson// This contains code to emit Builtin calls as LLVM code. 11022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson// 12022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson//===----------------------------------------------------------------------===// 13022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson 14022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson#include "CodeGenFunction.h" 1555bcace250e1ff366e4482714b344b8cbc8be5f3Fariborz Jahanian#include "CGObjCRuntime.h" 1655fc873017f10f6f566b182b70f6fc22aefa3464Chandler Carruth#include "CodeGenModule.h" 1755fc873017f10f6f566b182b70f6fc22aefa3464Chandler Carruth#include "TargetInfo.h" 18bef20ac367a09555b30d6eb3847a81ec164caf88Chris Lattner#include "clang/AST/ASTContext.h" 19c4a1dea2dc56bd1357ec91b829a0b9e68229a13eDaniel Dunbar#include "clang/AST/Decl.h" 206b15cdc1312f8fc45c86ee75e2a85106700e97f6Chris Lattner#include "clang/Basic/TargetBuiltins.h" 2155fc873017f10f6f566b182b70f6fc22aefa3464Chandler Carruth#include "clang/Basic/TargetInfo.h" 228b54999a831bb195c08541ca995ef0505c96193fMark Lacey#include "clang/CodeGen/CGFunctionInfo.h" 233b844ba7d5be205a9b4f5f0b0d1b7978977f4b8cChandler Carruth#include "llvm/IR/DataLayout.h" 243b844ba7d5be205a9b4f5f0b0d1b7978977f4b8cChandler Carruth#include "llvm/IR/Intrinsics.h" 25558229f267d2cdee69639df04382082e7c7ad54cJakub Staszak 26022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlssonusing namespace clang; 27022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlssonusing namespace CodeGen; 28ca6fcfad547dcec3fdd17790b4fab0918df74b37Anders Carlssonusing namespace llvm; 29ca6fcfad547dcec3fdd17790b4fab0918df74b37Anders Carlsson 30a45680b7e7c49ea9893c6cff585984f3e4120366John McCall/// getBuiltinLibFunction - Given a builtin id for a function like 31a45680b7e7c49ea9893c6cff585984f3e4120366John McCall/// "__builtin_fabsf", return a Function* for "fabsf". 32a45680b7e7c49ea9893c6cff585984f3e4120366John McCallllvm::Value *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD, 33a45680b7e7c49ea9893c6cff585984f3e4120366John McCall unsigned BuiltinID) { 34a45680b7e7c49ea9893c6cff585984f3e4120366John McCall assert(Context.BuiltinInfo.isLibFunction(BuiltinID)); 35a45680b7e7c49ea9893c6cff585984f3e4120366John McCall 36a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // Get the name, skip over the __builtin_ prefix (if necessary). 37a45680b7e7c49ea9893c6cff585984f3e4120366John McCall StringRef Name; 38a45680b7e7c49ea9893c6cff585984f3e4120366John McCall GlobalDecl D(FD); 39a45680b7e7c49ea9893c6cff585984f3e4120366John McCall 40a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // If the builtin has been declared explicitly with an assembler label, 41a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // use the mangled name. This differs from the plain label on platforms 42a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // that prefix labels. 43a45680b7e7c49ea9893c6cff585984f3e4120366John McCall if (FD->hasAttr<AsmLabelAttr>()) 44a45680b7e7c49ea9893c6cff585984f3e4120366John McCall Name = getMangledName(D); 45a45680b7e7c49ea9893c6cff585984f3e4120366John McCall else 46a45680b7e7c49ea9893c6cff585984f3e4120366John McCall Name = Context.BuiltinInfo.GetName(BuiltinID) + 10; 47a45680b7e7c49ea9893c6cff585984f3e4120366John McCall 48a45680b7e7c49ea9893c6cff585984f3e4120366John McCall llvm::FunctionType *Ty = 49a45680b7e7c49ea9893c6cff585984f3e4120366John McCall cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType())); 50a45680b7e7c49ea9893c6cff585984f3e4120366John McCall 51a45680b7e7c49ea9893c6cff585984f3e4120366John McCall return GetOrCreateLLVMFunction(Name, Ty, D, /*ForVTable=*/false); 52a45680b7e7c49ea9893c6cff585984f3e4120366John McCall} 53a45680b7e7c49ea9893c6cff585984f3e4120366John McCall 5426815d97c5743481e317f17a8d53a6819d061862John McCall/// Emit the conversions required to turn the given value into an 5526815d97c5743481e317f17a8d53a6819d061862John McCall/// integer of the given size. 5626815d97c5743481e317f17a8d53a6819d061862John McCallstatic Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V, 572acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner QualType T, llvm::IntegerType *IntType) { 5826815d97c5743481e317f17a8d53a6819d061862John McCall V = CGF.EmitToMemory(V, T); 5926815d97c5743481e317f17a8d53a6819d061862John McCall 6026815d97c5743481e317f17a8d53a6819d061862John McCall if (V->getType()->isPointerTy()) 6126815d97c5743481e317f17a8d53a6819d061862John McCall return CGF.Builder.CreatePtrToInt(V, IntType); 6226815d97c5743481e317f17a8d53a6819d061862John McCall 6326815d97c5743481e317f17a8d53a6819d061862John McCall assert(V->getType() == IntType); 6426815d97c5743481e317f17a8d53a6819d061862John McCall return V; 65db4325b098eff5e9e660db19f0148423fb21f27fChandler Carruth} 66db4325b098eff5e9e660db19f0148423fb21f27fChandler Carruth 6726815d97c5743481e317f17a8d53a6819d061862John McCallstatic Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V, 682acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner QualType T, llvm::Type *ResultType) { 6926815d97c5743481e317f17a8d53a6819d061862John McCall V = CGF.EmitFromMemory(V, T); 7026815d97c5743481e317f17a8d53a6819d061862John McCall 7126815d97c5743481e317f17a8d53a6819d061862John McCall if (ResultType->isPointerTy()) 7226815d97c5743481e317f17a8d53a6819d061862John McCall return CGF.Builder.CreateIntToPtr(V, ResultType); 7326815d97c5743481e317f17a8d53a6819d061862John McCall 7426815d97c5743481e317f17a8d53a6819d061862John McCall assert(V->getType() == ResultType); 7526815d97c5743481e317f17a8d53a6819d061862John McCall return V; 76db4325b098eff5e9e660db19f0148423fb21f27fChandler Carruth} 77db4325b098eff5e9e660db19f0148423fb21f27fChandler Carruth 780002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar/// Utility to insert an atomic instruction based on Instrinsic::ID 790002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar/// and the expression node. 80cb61a7bbe635cfa941b1aeaaa1fbda1bf900ee51Daniel Dunbarstatic RValue EmitBinaryAtomic(CodeGenFunction &CGF, 81c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman llvm::AtomicRMWInst::BinOp Kind, 82c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman const CallExpr *E) { 8326815d97c5743481e317f17a8d53a6819d061862John McCall QualType T = E->getType(); 8426815d97c5743481e317f17a8d53a6819d061862John McCall assert(E->getArg(0)->getType()->isPointerType()); 8526815d97c5743481e317f17a8d53a6819d061862John McCall assert(CGF.getContext().hasSameUnqualifiedType(T, 8626815d97c5743481e317f17a8d53a6819d061862John McCall E->getArg(0)->getType()->getPointeeType())); 8726815d97c5743481e317f17a8d53a6819d061862John McCall assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType())); 8826815d97c5743481e317f17a8d53a6819d061862John McCall 894f209445c06a43283c6f72dda7c925538b1578e9Chris Lattner llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0)); 90956a5a17713deb1b5b27893303c4f308a1bd2a62Micah Villmow unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); 9126815d97c5743481e317f17a8d53a6819d061862John McCall 929cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::IntegerType *IntType = 93db4325b098eff5e9e660db19f0148423fb21f27fChandler Carruth llvm::IntegerType::get(CGF.getLLVMContext(), 9426815d97c5743481e317f17a8d53a6819d061862John McCall CGF.getContext().getTypeSize(T)); 959cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); 9626815d97c5743481e317f17a8d53a6819d061862John McCall 9726815d97c5743481e317f17a8d53a6819d061862John McCall llvm::Value *Args[2]; 9826815d97c5743481e317f17a8d53a6819d061862John McCall Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType); 9926815d97c5743481e317f17a8d53a6819d061862John McCall Args[1] = CGF.EmitScalarExpr(E->getArg(1)); 1002acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ValueType = Args[1]->getType(); 10126815d97c5743481e317f17a8d53a6819d061862John McCall Args[1] = EmitToInt(CGF, Args[1], T, IntType); 10226815d97c5743481e317f17a8d53a6819d061862John McCall 103c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman llvm::Value *Result = 104c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman CGF.Builder.CreateAtomicRMW(Kind, Args[0], Args[1], 105c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman llvm::SequentiallyConsistent); 10626815d97c5743481e317f17a8d53a6819d061862John McCall Result = EmitFromInt(CGF, Result, T, ValueType); 10726815d97c5743481e317f17a8d53a6819d061862John McCall return RValue::get(Result); 1080002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar} 1090002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar 1100002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar/// Utility to insert an atomic instruction based Instrinsic::ID and 11126815d97c5743481e317f17a8d53a6819d061862John McCall/// the expression node, where the return value is the result of the 11226815d97c5743481e317f17a8d53a6819d061862John McCall/// operation. 113420b11850d3f4557421f43f519b59d528329c668Chris Lattnerstatic RValue EmitBinaryAtomicPost(CodeGenFunction &CGF, 114c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman llvm::AtomicRMWInst::BinOp Kind, 115c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman const CallExpr *E, 116176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Instruction::BinaryOps Op, 117176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines bool Invert = false) { 11826815d97c5743481e317f17a8d53a6819d061862John McCall QualType T = E->getType(); 11926815d97c5743481e317f17a8d53a6819d061862John McCall assert(E->getArg(0)->getType()->isPointerType()); 12026815d97c5743481e317f17a8d53a6819d061862John McCall assert(CGF.getContext().hasSameUnqualifiedType(T, 12126815d97c5743481e317f17a8d53a6819d061862John McCall E->getArg(0)->getType()->getPointeeType())); 12226815d97c5743481e317f17a8d53a6819d061862John McCall assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType())); 12326815d97c5743481e317f17a8d53a6819d061862John McCall 1244f209445c06a43283c6f72dda7c925538b1578e9Chris Lattner llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0)); 125956a5a17713deb1b5b27893303c4f308a1bd2a62Micah Villmow unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); 12626815d97c5743481e317f17a8d53a6819d061862John McCall 1279cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::IntegerType *IntType = 128db4325b098eff5e9e660db19f0148423fb21f27fChandler Carruth llvm::IntegerType::get(CGF.getLLVMContext(), 12926815d97c5743481e317f17a8d53a6819d061862John McCall CGF.getContext().getTypeSize(T)); 1309cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); 13126815d97c5743481e317f17a8d53a6819d061862John McCall 13226815d97c5743481e317f17a8d53a6819d061862John McCall llvm::Value *Args[2]; 13326815d97c5743481e317f17a8d53a6819d061862John McCall Args[1] = CGF.EmitScalarExpr(E->getArg(1)); 1342acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ValueType = Args[1]->getType(); 13526815d97c5743481e317f17a8d53a6819d061862John McCall Args[1] = EmitToInt(CGF, Args[1], T, IntType); 13626815d97c5743481e317f17a8d53a6819d061862John McCall Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType); 13726815d97c5743481e317f17a8d53a6819d061862John McCall 138c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman llvm::Value *Result = 139c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman CGF.Builder.CreateAtomicRMW(Kind, Args[0], Args[1], 140c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman llvm::SequentiallyConsistent); 14126815d97c5743481e317f17a8d53a6819d061862John McCall Result = CGF.Builder.CreateBinOp(Op, Result, Args[1]); 142176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (Invert) 143176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Result = CGF.Builder.CreateBinOp(llvm::Instruction::Xor, Result, 144176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::ConstantInt::get(IntType, -1)); 14526815d97c5743481e317f17a8d53a6819d061862John McCall Result = EmitFromInt(CGF, Result, T, ValueType); 14626815d97c5743481e317f17a8d53a6819d061862John McCall return RValue::get(Result); 1471ffe281890f3cd7728316b45a1f3dd4d3120af7bMon P Wang} 1481ffe281890f3cd7728316b45a1f3dd4d3120af7bMon P Wang 149176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines/// EmitFAbs - Emit a call to @llvm.fabs(). 150176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hinesstatic Value *EmitFAbs(CodeGenFunction &CGF, Value *V) { 151176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *F = CGF.CGM.getIntrinsic(Intrinsic::fabs, V->getType()); 152176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::CallInst *Call = CGF.Builder.CreateCall(F, V); 153176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Call->setDoesNotAccessMemory(); 154176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return Call; 155420b11850d3f4557421f43f519b59d528329c668Chris Lattner} 156420b11850d3f4557421f43f519b59d528329c668Chris Lattner 157a45680b7e7c49ea9893c6cff585984f3e4120366John McCallstatic RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *Fn, 158a45680b7e7c49ea9893c6cff585984f3e4120366John McCall const CallExpr *E, llvm::Value *calleeValue) { 159176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return CGF.EmitCall(E->getCallee()->getType(), calleeValue, E, 160176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines ReturnValueSlot(), Fn); 161a45680b7e7c49ea9893c6cff585984f3e4120366John McCall} 162a45680b7e7c49ea9893c6cff585984f3e4120366John McCall 1630cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// \brief Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.* 1640cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// depending on IntrinsicID. 1650cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// 1660cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// \arg CGF The current codegen function. 1670cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// \arg IntrinsicID The ID for the Intrinsic we wish to generate. 1680cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// \arg X The first argument to the llvm.*.with.overflow.*. 1690cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// \arg Y The second argument to the llvm.*.with.overflow.*. 1700cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// \arg Carry The carry returned by the llvm.*.with.overflow.*. 1710cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// \returns The result (i.e. sum/product) returned by the intrinsic. 1720cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesmanstatic llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF, 1730cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman const llvm::Intrinsic::ID IntrinsicID, 1740cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *X, llvm::Value *Y, 1750cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *&Carry) { 1760cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // Make sure we have integers of the same width. 1770cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman assert(X->getType() == Y->getType() && 1780cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman "Arguments must be the same type. (Did you forget to make sure both " 1790cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman "arguments have the same integer width?)"); 1800cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman 181563fb903fab15972e2d9c66b0ae046a94be86a71NAKAMURA Takumi llvm::Value *Callee = CGF.CGM.getIntrinsic(IntrinsicID, X->getType()); 1820cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *Tmp = CGF.Builder.CreateCall2(Callee, X, Y); 1830cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman Carry = CGF.Builder.CreateExtractValue(Tmp, 1); 1840cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman return CGF.Builder.CreateExtractValue(Tmp, 0); 1850cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman} 1860cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman 1871eb4433ac451dc16f4133a88af2d002ac26c58efMike StumpRValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, 188ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar unsigned BuiltinID, const CallExpr *E) { 189564ea2a99b3afeac9ded332730a56db1f6358a58Chris Lattner // See if we can constant fold this builtin. If so, don't emit it at all. 190f35d35a2316dcb65d078844696c2032b71a7f103Anders Carlsson Expr::EvalResult Result; 19152a27f5be98d99fd5339949d8a3d0b2aec655e0bEli Friedman if (E->EvaluateAsRValue(Result, CGM.getContext()) && 192dd697bc8629f0fa6777245610d52ca7ceb3b67f4Fariborz Jahanian !Result.hasSideEffects()) { 193f35d35a2316dcb65d078844696c2032b71a7f103Anders Carlsson if (Result.Val.isInt()) 194d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall return RValue::get(llvm::ConstantInt::get(getLLVMContext(), 1954a28d5deeba33722aa009eab488591fb9055cc7eOwen Anderson Result.Val.getInt())); 196a1aa9e36e6e21f74c56cf9e72cb5bd9aa2a92fd4Chris Lattner if (Result.Val.isFloat()) 197d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall return RValue::get(llvm::ConstantFP::get(getLLVMContext(), 198d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Result.Val.getFloat())); 1991f32999ec79a980576e100d64d5f3267eb19ea49Chris Lattner } 2001eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 201564ea2a99b3afeac9ded332730a56db1f6358a58Chris Lattner switch (BuiltinID) { 202564ea2a99b3afeac9ded332730a56db1f6358a58Chris Lattner default: break; // Handle intrinsics and libm functions below. 203506ff88f44562df267b6a06608ab841b76df2a2bChris Lattner case Builtin::BI__builtin___CFStringMakeConstantString: 2040d13f6fdbdd6f06e2449b8834dda53334abd399aDavid Chisnall case Builtin::BI__builtin___NSStringMakeConstantString: 2056bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(CGM.EmitConstantExpr(E, E->getType(), nullptr)); 2066a705f0443f8398ece14d80e71d8e8c9e71aa84aChris Lattner case Builtin::BI__builtin_stdarg_start: 207793680ed8104bf088d1b382b963a8badcb3f07deAnders Carlsson case Builtin::BI__builtin_va_start: 208651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case Builtin::BI__va_start: 209793680ed8104bf088d1b382b963a8badcb3f07deAnders Carlsson case Builtin::BI__builtin_va_end: { 210651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *ArgValue = (BuiltinID == Builtin::BI__va_start) 211651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ? EmitScalarExpr(E->getArg(0)) 212651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : EmitVAListRef(E->getArg(0)); 2132acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *DestType = Int8PtrTy; 214793680ed8104bf088d1b382b963a8badcb3f07deAnders Carlsson if (ArgValue->getType() != DestType) 2151eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump ArgValue = Builder.CreateBitCast(ArgValue, DestType, 216b27ffbef8e8aa1e87b63cbc0d9cd630aba30dae5Daniel Dunbar ArgValue->getName().data()); 217793680ed8104bf088d1b382b963a8badcb3f07deAnders Carlsson 2181eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump Intrinsic::ID inst = (BuiltinID == Builtin::BI__builtin_va_end) ? 2196a705f0443f8398ece14d80e71d8e8c9e71aa84aChris Lattner Intrinsic::vaend : Intrinsic::vastart; 2207acda7c4a0e4aec6c003b3169ca45a5f3bc7e033Chris Lattner return RValue::get(Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue)); 221793680ed8104bf088d1b382b963a8badcb3f07deAnders Carlsson } 222a28ef8b5c6ea452472d89041128fac8b683fe968Anders Carlsson case Builtin::BI__builtin_va_copy: { 2234fd0aa5803357d8c72eeac2cae15e12649ea08feEli Friedman Value *DstPtr = EmitVAListRef(E->getArg(0)); 2244fd0aa5803357d8c72eeac2cae15e12649ea08feEli Friedman Value *SrcPtr = EmitVAListRef(E->getArg(1)); 225a28ef8b5c6ea452472d89041128fac8b683fe968Anders Carlsson 2262acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *Type = Int8PtrTy; 227a28ef8b5c6ea452472d89041128fac8b683fe968Anders Carlsson 228a28ef8b5c6ea452472d89041128fac8b683fe968Anders Carlsson DstPtr = Builder.CreateBitCast(DstPtr, Type); 229a28ef8b5c6ea452472d89041128fac8b683fe968Anders Carlsson SrcPtr = Builder.CreateBitCast(SrcPtr, Type); 2301eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump return RValue::get(Builder.CreateCall2(CGM.getIntrinsic(Intrinsic::vacopy), 2313eae03e7d165f20a863a9a4d7122ba2a691ab16dChris Lattner DstPtr, SrcPtr)); 232a28ef8b5c6ea452472d89041128fac8b683fe968Anders Carlsson } 233258f930227c1a102c9c22eee88df65f748863425Jim Grosbach case Builtin::BI__builtin_abs: 234f7b2d8b3b1971e73e2d2f0662520dc7693235a7fEli Friedman case Builtin::BI__builtin_labs: 235f7b2d8b3b1971e73e2d2f0662520dc7693235a7fEli Friedman case Builtin::BI__builtin_llabs: { 2361eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump Value *ArgValue = EmitScalarExpr(E->getArg(0)); 2371eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2389a847f598c4907a72b8593b364b9e6b94b086e75Chris Lattner Value *NegOp = Builder.CreateNeg(ArgValue, "neg"); 2391eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump Value *CmpResult = 2401eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump Builder.CreateICmpSGE(ArgValue, 241c9c88b4159791c48e486ca94e3743b5979e2b7a6Owen Anderson llvm::Constant::getNullValue(ArgValue->getType()), 2429a847f598c4907a72b8593b364b9e6b94b086e75Chris Lattner "abscond"); 2431eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump Value *Result = 244c2251dc59b0edc28f9303637dec970a7520939adAnders Carlsson Builder.CreateSelect(CmpResult, ArgValue, NegOp, "abs"); 2451eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 246c2251dc59b0edc28f9303637dec970a7520939adAnders Carlsson return RValue::get(Result); 247c2251dc59b0edc28f9303637dec970a7520939adAnders Carlsson } 248176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__builtin_fabs: 249176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__builtin_fabsf: 250176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__builtin_fabsl: { 251176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *Arg1 = EmitScalarExpr(E->getArg(0)); 252176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *Result = EmitFAbs(*this, Arg1); 253176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return RValue::get(Result); 254176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 255176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__builtin_fmod: 256176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__builtin_fmodf: 257176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__builtin_fmodl: { 258176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *Arg1 = EmitScalarExpr(E->getArg(0)); 259176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *Arg2 = EmitScalarExpr(E->getArg(1)); 260176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *Result = Builder.CreateFRem(Arg1, Arg2, "fmod"); 261176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return RValue::get(Result); 262176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 263258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 264ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian case Builtin::BI__builtin_conj: 265ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian case Builtin::BI__builtin_conjf: 266ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian case Builtin::BI__builtin_conjl: { 267ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0)); 268ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian Value *Real = ComplexVal.first; 269ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian Value *Imag = ComplexVal.second; 270258f930227c1a102c9c22eee88df65f748863425Jim Grosbach Value *Zero = 271258f930227c1a102c9c22eee88df65f748863425Jim Grosbach Imag->getType()->isFPOrFPVectorTy() 272ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian ? llvm::ConstantFP::getZeroValueForNegation(Imag->getType()) 273ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian : llvm::Constant::getNullValue(Imag->getType()); 274258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 275ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian Imag = Builder.CreateFSub(Zero, Imag, "sub"); 276ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian return RValue::getComplex(std::make_pair(Real, Imag)); 277ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian } 278ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian case Builtin::BI__builtin_creal: 279ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian case Builtin::BI__builtin_crealf: 28008cc03fcd97c3886b004d861aadbc28de70092c5Meador Inge case Builtin::BI__builtin_creall: 28108cc03fcd97c3886b004d861aadbc28de70092c5Meador Inge case Builtin::BIcreal: 28208cc03fcd97c3886b004d861aadbc28de70092c5Meador Inge case Builtin::BIcrealf: 28308cc03fcd97c3886b004d861aadbc28de70092c5Meador Inge case Builtin::BIcreall: { 284ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0)); 285ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian return RValue::get(ComplexVal.first); 286ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian } 287258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 288ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian case Builtin::BI__builtin_cimag: 289ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian case Builtin::BI__builtin_cimagf: 29008cc03fcd97c3886b004d861aadbc28de70092c5Meador Inge case Builtin::BI__builtin_cimagl: 29108cc03fcd97c3886b004d861aadbc28de70092c5Meador Inge case Builtin::BIcimag: 29208cc03fcd97c3886b004d861aadbc28de70092c5Meador Inge case Builtin::BIcimagf: 29308cc03fcd97c3886b004d861aadbc28de70092c5Meador Inge case Builtin::BIcimagl: { 294ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0)); 295ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian return RValue::get(ComplexVal.second); 296ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian } 297258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 298a49a2839266f0bac2b6286e9f49fdc0c292938feBenjamin Kramer case Builtin::BI__builtin_ctzs: 2993a31d60cffedfb7c9e6d129a5c9ba15fa74f179aAnders Carlsson case Builtin::BI__builtin_ctz: 3003a31d60cffedfb7c9e6d129a5c9ba15fa74f179aAnders Carlsson case Builtin::BI__builtin_ctzl: 3013a31d60cffedfb7c9e6d129a5c9ba15fa74f179aAnders Carlsson case Builtin::BI__builtin_ctzll: { 3023a31d60cffedfb7c9e6d129a5c9ba15fa74f179aAnders Carlsson Value *ArgValue = EmitScalarExpr(E->getArg(0)); 3031eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3049cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *ArgType = ArgValue->getType(); 3058dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType); 3063a31d60cffedfb7c9e6d129a5c9ba15fa74f179aAnders Carlsson 3072acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ResultType = ConvertType(E->getType()); 30864aa4b3ec7e62288e2e66c1935487ece995ca94bJohn McCall Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef()); 3098b30a9379f730875ba8fb2d0fe2d43611e0c20ffBob Wilson Value *Result = Builder.CreateCall2(F, ArgValue, ZeroUndef); 3103a31d60cffedfb7c9e6d129a5c9ba15fa74f179aAnders Carlsson if (Result->getType() != ResultType) 311eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 312eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands "cast"); 3133a31d60cffedfb7c9e6d129a5c9ba15fa74f179aAnders Carlsson return RValue::get(Result); 3143a31d60cffedfb7c9e6d129a5c9ba15fa74f179aAnders Carlsson } 315a49a2839266f0bac2b6286e9f49fdc0c292938feBenjamin Kramer case Builtin::BI__builtin_clzs: 316f4e853340590d5c32e58379e8c379ea1777d3101Eli Friedman case Builtin::BI__builtin_clz: 317f4e853340590d5c32e58379e8c379ea1777d3101Eli Friedman case Builtin::BI__builtin_clzl: 318f4e853340590d5c32e58379e8c379ea1777d3101Eli Friedman case Builtin::BI__builtin_clzll: { 319f4e853340590d5c32e58379e8c379ea1777d3101Eli Friedman Value *ArgValue = EmitScalarExpr(E->getArg(0)); 3201eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3219cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *ArgType = ArgValue->getType(); 3228dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType); 323f4e853340590d5c32e58379e8c379ea1777d3101Eli Friedman 3242acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ResultType = ConvertType(E->getType()); 32564aa4b3ec7e62288e2e66c1935487ece995ca94bJohn McCall Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef()); 3268b30a9379f730875ba8fb2d0fe2d43611e0c20ffBob Wilson Value *Result = Builder.CreateCall2(F, ArgValue, ZeroUndef); 327f4e853340590d5c32e58379e8c379ea1777d3101Eli Friedman if (Result->getType() != ResultType) 328eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 329eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands "cast"); 330f4e853340590d5c32e58379e8c379ea1777d3101Eli Friedman return RValue::get(Result); 331f4e853340590d5c32e58379e8c379ea1777d3101Eli Friedman } 33204b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_ffs: 33304b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_ffsl: 33404b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_ffsll: { 33504b290030eee33295600728450f348989d1a627eDaniel Dunbar // ffs(x) -> x ? cttz(x) + 1 : 0 33604b290030eee33295600728450f348989d1a627eDaniel Dunbar Value *ArgValue = EmitScalarExpr(E->getArg(0)); 3371eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3389cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *ArgType = ArgValue->getType(); 3398dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType); 3401eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3412acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ResultType = ConvertType(E->getType()); 34250058ec63f38eabeb94391a61d2e7fb18a062173Chandler Carruth Value *Tmp = Builder.CreateAdd(Builder.CreateCall2(F, ArgValue, 34350058ec63f38eabeb94391a61d2e7fb18a062173Chandler Carruth Builder.getTrue()), 344578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer llvm::ConstantInt::get(ArgType, 1)); 345c9c88b4159791c48e486ca94e3743b5979e2b7a6Owen Anderson Value *Zero = llvm::Constant::getNullValue(ArgType); 34604b290030eee33295600728450f348989d1a627eDaniel Dunbar Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero"); 34704b290030eee33295600728450f348989d1a627eDaniel Dunbar Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp, "ffs"); 34804b290030eee33295600728450f348989d1a627eDaniel Dunbar if (Result->getType() != ResultType) 349eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 350eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands "cast"); 35104b290030eee33295600728450f348989d1a627eDaniel Dunbar return RValue::get(Result); 35204b290030eee33295600728450f348989d1a627eDaniel Dunbar } 35304b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_parity: 35404b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_parityl: 35504b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_parityll: { 35604b290030eee33295600728450f348989d1a627eDaniel Dunbar // parity(x) -> ctpop(x) & 1 35704b290030eee33295600728450f348989d1a627eDaniel Dunbar Value *ArgValue = EmitScalarExpr(E->getArg(0)); 3581eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3599cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *ArgType = ArgValue->getType(); 3608dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType); 3611eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3622acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ResultType = ConvertType(E->getType()); 363578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer Value *Tmp = Builder.CreateCall(F, ArgValue); 364578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1)); 36504b290030eee33295600728450f348989d1a627eDaniel Dunbar if (Result->getType() != ResultType) 366eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 367eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands "cast"); 36804b290030eee33295600728450f348989d1a627eDaniel Dunbar return RValue::get(Result); 36904b290030eee33295600728450f348989d1a627eDaniel Dunbar } 37004b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_popcount: 37104b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_popcountl: 37204b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_popcountll: { 37304b290030eee33295600728450f348989d1a627eDaniel Dunbar Value *ArgValue = EmitScalarExpr(E->getArg(0)); 3741eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3759cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *ArgType = ArgValue->getType(); 3768dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType); 3771eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3782acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ResultType = ConvertType(E->getType()); 379578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer Value *Result = Builder.CreateCall(F, ArgValue); 38004b290030eee33295600728450f348989d1a627eDaniel Dunbar if (Result->getType() != ResultType) 381eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 382eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands "cast"); 38304b290030eee33295600728450f348989d1a627eDaniel Dunbar return RValue::get(Result); 38404b290030eee33295600728450f348989d1a627eDaniel Dunbar } 385e42b8a596886fc98e367c73e54d761446700029eFariborz Jahanian case Builtin::BI__builtin_expect: { 386dd697bc8629f0fa6777245610d52ca7ceb3b67f4Fariborz Jahanian Value *ArgValue = EmitScalarExpr(E->getArg(0)); 3879cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *ArgType = ArgValue->getType(); 388558229f267d2cdee69639df04382082e7c7ad54cJakub Staszak 3898dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType); 390558229f267d2cdee69639df04382082e7c7ad54cJakub Staszak Value *ExpectedValue = EmitScalarExpr(E->getArg(1)); 391558229f267d2cdee69639df04382082e7c7ad54cJakub Staszak 392558229f267d2cdee69639df04382082e7c7ad54cJakub Staszak Value *Result = Builder.CreateCall2(FnExpect, ArgValue, ExpectedValue, 393558229f267d2cdee69639df04382082e7c7ad54cJakub Staszak "expval"); 394558229f267d2cdee69639df04382082e7c7ad54cJakub Staszak return RValue::get(Result); 395e42b8a596886fc98e367c73e54d761446700029eFariborz Jahanian } 396176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__builtin_assume_aligned: { 397176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *PtrValue = EmitScalarExpr(E->getArg(0)); 398176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *OffsetValue = 399176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) : nullptr; 400176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 401176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *AlignmentValue = EmitScalarExpr(E->getArg(1)); 402176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue); 403176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines unsigned Alignment = (unsigned) AlignmentCI->getZExtValue(); 404176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 405176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines EmitAlignmentAssumption(PtrValue, Alignment, OffsetValue); 406176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return RValue::get(PtrValue); 407176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 408176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__assume: 409176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__builtin_assume: { 410176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (E->getArg(0)->HasSideEffects(getContext())) 411176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return RValue::get(nullptr); 412176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 413176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *ArgValue = EmitScalarExpr(E->getArg(0)); 414176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *FnAssume = CGM.getIntrinsic(Intrinsic::assume); 415176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return RValue::get(Builder.CreateCall(FnAssume, ArgValue)); 416176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 417d190057934331390ff67ebf51d66186dd5e392f0Benjamin Kramer case Builtin::BI__builtin_bswap16: 418df4852ac816e6050d53b808b86d7c1c9738eb99eAnders Carlsson case Builtin::BI__builtin_bswap32: 419df4852ac816e6050d53b808b86d7c1c9738eb99eAnders Carlsson case Builtin::BI__builtin_bswap64: { 4201feedd84221e8dbcc3faf3de27cc42b559db845dChris Lattner Value *ArgValue = EmitScalarExpr(E->getArg(0)); 4219cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *ArgType = ArgValue->getType(); 4228dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::bswap, ArgType); 423578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer return RValue::get(Builder.CreateCall(F, ArgValue)); 4241eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } 425d5f8a4fd4d6dfb0415b93bb7ab721bba5cab1332Daniel Dunbar case Builtin::BI__builtin_object_size: { 426c6794850a570a91c5f224b6f0293db9f560f4213Richard Smith // We rely on constant folding to deal with expressions with side effects. 427c6794850a570a91c5f224b6f0293db9f560f4213Richard Smith assert(!E->getArg(0)->HasSideEffects(getContext()) && 428c6794850a570a91c5f224b6f0293db9f560f4213Richard Smith "should have been constant folded"); 429c6794850a570a91c5f224b6f0293db9f560f4213Richard Smith 430b16d32f74ffc467a5604934a1f844906be20cf7dMike Stump // We pass this builtin onto the optimizer so that it can 431b16d32f74ffc467a5604934a1f844906be20cf7dMike Stump // figure out the object size in more complex cases. 4328dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer llvm::Type *ResType = ConvertType(E->getType()); 433258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 434fee667f35e64751baa7fefe70b4e7bab06c8cd86Eric Christopher // LLVM only supports 0 and 2, make sure that we pass along that 435fee667f35e64751baa7fefe70b4e7bab06c8cd86Eric Christopher // as a boolean. 436fee667f35e64751baa7fefe70b4e7bab06c8cd86Eric Christopher Value *Ty = EmitScalarExpr(E->getArg(1)); 437fee667f35e64751baa7fefe70b4e7bab06c8cd86Eric Christopher ConstantInt *CI = dyn_cast<ConstantInt>(Ty); 438fee667f35e64751baa7fefe70b4e7bab06c8cd86Eric Christopher assert(CI); 439fee667f35e64751baa7fefe70b4e7bab06c8cd86Eric Christopher uint64_t val = CI->getZExtValue(); 440258f930227c1a102c9c22eee88df65f748863425Jim Grosbach CI = ConstantInt::get(Builder.getInt1Ty(), (val & 0x2) >> 1); 4418dd4baacf584b1fc7b0e2c79a0b8b197a4fb5fabMatt Arsenault // FIXME: Get right address space. 4428dd4baacf584b1fc7b0e2c79a0b8b197a4fb5fabMatt Arsenault llvm::Type *Tys[] = { ResType, Builder.getInt8PtrTy(0) }; 4438dd4baacf584b1fc7b0e2c79a0b8b197a4fb5fabMatt Arsenault Value *F = CGM.getIntrinsic(Intrinsic::objectsize, Tys); 4443e86a0433db4c664d29f2b19eb977138e071a68aNuno Lopes return RValue::get(Builder.CreateCall2(F, EmitScalarExpr(E->getArg(0)),CI)); 445d5f8a4fd4d6dfb0415b93bb7ab721bba5cab1332Daniel Dunbar } 4464493f79fce48cd9cbd9f55fa9d452cde736747a0Daniel Dunbar case Builtin::BI__builtin_prefetch: { 4474493f79fce48cd9cbd9f55fa9d452cde736747a0Daniel Dunbar Value *Locality, *RW, *Address = EmitScalarExpr(E->getArg(0)); 4484493f79fce48cd9cbd9f55fa9d452cde736747a0Daniel Dunbar // FIXME: Technically these constants should of type 'int', yes? 4491eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump RW = (E->getNumArgs() > 1) ? EmitScalarExpr(E->getArg(1)) : 45077b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner llvm::ConstantInt::get(Int32Ty, 0); 4511eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump Locality = (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) : 45277b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner llvm::ConstantInt::get(Int32Ty, 3); 4532eccb672799d19fb535ce349566fea6729cbb891Bruno Cardoso Lopes Value *Data = llvm::ConstantInt::get(Int32Ty, 1); 4548dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::prefetch); 4552eccb672799d19fb535ce349566fea6729cbb891Bruno Cardoso Lopes return RValue::get(Builder.CreateCall4(F, Address, RW, Locality, Data)); 4564493f79fce48cd9cbd9f55fa9d452cde736747a0Daniel Dunbar } 457a841c19f7860393d6319bf40e9d662284462771dHal Finkel case Builtin::BI__builtin_readcyclecounter: { 458a841c19f7860393d6319bf40e9d662284462771dHal Finkel Value *F = CGM.getIntrinsic(Intrinsic::readcyclecounter); 459a841c19f7860393d6319bf40e9d662284462771dHal Finkel return RValue::get(Builder.CreateCall(F)); 460a841c19f7860393d6319bf40e9d662284462771dHal Finkel } 461651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case Builtin::BI__builtin___clear_cache: { 462651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Begin = EmitScalarExpr(E->getArg(0)); 463651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *End = EmitScalarExpr(E->getArg(1)); 464651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *F = CGM.getIntrinsic(Intrinsic::clear_cache); 465651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return RValue::get(Builder.CreateCall2(F, Begin, End)); 466651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4674493f79fce48cd9cbd9f55fa9d452cde736747a0Daniel Dunbar case Builtin::BI__builtin_trap: { 4688dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::trap); 4694493f79fce48cd9cbd9f55fa9d452cde736747a0Daniel Dunbar return RValue::get(Builder.CreateCall(F)); 470df4852ac816e6050d53b808b86d7c1c9738eb99eAnders Carlsson } 471385cc5f01b59af0183a825351d93695ca7185091Nico Weber case Builtin::BI__debugbreak: { 472385cc5f01b59af0183a825351d93695ca7185091Nico Weber Value *F = CGM.getIntrinsic(Intrinsic::debugtrap); 473385cc5f01b59af0183a825351d93695ca7185091Nico Weber return RValue::get(Builder.CreateCall(F)); 474385cc5f01b59af0183a825351d93695ca7185091Nico Weber } 47521190d54634d6e244e85d28ad915ce2fe86ecbffChris Lattner case Builtin::BI__builtin_unreachable: { 476176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (SanOpts.has(SanitizerKind::Unreachable)) { 477176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines SanitizerScope SanScope(this); 478176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines EmitCheck(std::make_pair(static_cast<llvm::Value *>(Builder.getFalse()), 479176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines SanitizerKind::Unreachable), 480176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines "builtin_unreachable", EmitCheckSourceLocation(E->getExprLoc()), 481176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines None); 482176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } else 483cd5b22e12b6513163dd131589746c194090f14e6John McCall Builder.CreateUnreachable(); 484cd5b22e12b6513163dd131589746c194090f14e6John McCall 485cd5b22e12b6513163dd131589746c194090f14e6John McCall // We do need to preserve an insertion point. 486d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall EmitBlock(createBasicBlock("unreachable.cont")); 487cd5b22e12b6513163dd131589746c194090f14e6John McCall 4886bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 48921190d54634d6e244e85d28ad915ce2fe86ecbffChris Lattner } 490258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 491a933c3c052bbd87b01cc6fc7a7745e1c4b1757fbDaniel Dunbar case Builtin::BI__builtin_powi: 492a933c3c052bbd87b01cc6fc7a7745e1c4b1757fbDaniel Dunbar case Builtin::BI__builtin_powif: 493a933c3c052bbd87b01cc6fc7a7745e1c4b1757fbDaniel Dunbar case Builtin::BI__builtin_powil: { 494a933c3c052bbd87b01cc6fc7a7745e1c4b1757fbDaniel Dunbar Value *Base = EmitScalarExpr(E->getArg(0)); 495a933c3c052bbd87b01cc6fc7a7745e1c4b1757fbDaniel Dunbar Value *Exponent = EmitScalarExpr(E->getArg(1)); 4969cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *ArgType = Base->getType(); 4978dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::powi, ArgType); 498578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer return RValue::get(Builder.CreateCall2(F, Base, Exponent)); 499a933c3c052bbd87b01cc6fc7a7745e1c4b1757fbDaniel Dunbar } 500a933c3c052bbd87b01cc6fc7a7745e1c4b1757fbDaniel Dunbar 501fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_isgreater: 502fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_isgreaterequal: 503fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_isless: 504fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_islessequal: 505fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_islessgreater: 506fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_isunordered: { 507fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner // Ordered comparisons: we know the arguments to these are matching scalar 508fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner // floating point values. 5091eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump Value *LHS = EmitScalarExpr(E->getArg(0)); 510fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner Value *RHS = EmitScalarExpr(E->getArg(1)); 5111eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 512fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner switch (BuiltinID) { 513b219cfc4d75f0a03630b7c4509ef791b7e97b2c8David Blaikie default: llvm_unreachable("Unknown ordered comparison"); 514fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_isgreater: 515fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner LHS = Builder.CreateFCmpOGT(LHS, RHS, "cmp"); 516fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner break; 517fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_isgreaterequal: 518fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner LHS = Builder.CreateFCmpOGE(LHS, RHS, "cmp"); 519fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner break; 520fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_isless: 521fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner LHS = Builder.CreateFCmpOLT(LHS, RHS, "cmp"); 522fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner break; 523fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_islessequal: 524fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner LHS = Builder.CreateFCmpOLE(LHS, RHS, "cmp"); 525fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner break; 526fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_islessgreater: 527fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner LHS = Builder.CreateFCmpONE(LHS, RHS, "cmp"); 528fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner break; 5291eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump case Builtin::BI__builtin_isunordered: 530fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner LHS = Builder.CreateFCmpUNO(LHS, RHS, "cmp"); 531fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner break; 532fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner } 533fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner // ZExt bool to int type. 534578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer return RValue::get(Builder.CreateZExt(LHS, ConvertType(E->getType()))); 535fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner } 536d6139895f43d161a972d134ffda4229d2f548eb6Eli Friedman case Builtin::BI__builtin_isnan: { 537d6139895f43d161a972d134ffda4229d2f548eb6Eli Friedman Value *V = EmitScalarExpr(E->getArg(0)); 538d6139895f43d161a972d134ffda4229d2f548eb6Eli Friedman V = Builder.CreateFCmpUNO(V, V, "cmp"); 539578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); 540d6139895f43d161a972d134ffda4229d2f548eb6Eli Friedman } 541258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 542420b11850d3f4557421f43f519b59d528329c668Chris Lattner case Builtin::BI__builtin_isinf: { 543420b11850d3f4557421f43f519b59d528329c668Chris Lattner // isinf(x) --> fabs(x) == infinity 544420b11850d3f4557421f43f519b59d528329c668Chris Lattner Value *V = EmitScalarExpr(E->getArg(0)); 545176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines V = EmitFAbs(*this, V); 546258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 547420b11850d3f4557421f43f519b59d528329c668Chris Lattner V = Builder.CreateFCmpOEQ(V, ConstantFP::getInfinity(V->getType()),"isinf"); 548578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); 549420b11850d3f4557421f43f519b59d528329c668Chris Lattner } 550258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 55158ae5b4b1e2fc02b95d6650af5755a59639aa153Chris Lattner // TODO: BI__builtin_isinf_sign 55258ae5b4b1e2fc02b95d6650af5755a59639aa153Chris Lattner // isinf_sign(x) -> isinf(x) ? (signbit(x) ? -1 : 1) : 0 5536349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer 5546349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer case Builtin::BI__builtin_isnormal: { 5556349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer // isnormal(x) --> x == x && fabsf(x) < infinity && fabsf(x) >= float_min 5566349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer Value *V = EmitScalarExpr(E->getArg(0)); 5576349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq"); 5586349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer 559176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *Abs = EmitFAbs(*this, V); 5606349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer Value *IsLessThanInf = 5616349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer Builder.CreateFCmpULT(Abs, ConstantFP::getInfinity(V->getType()),"isinf"); 5626349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer APFloat Smallest = APFloat::getSmallestNormalized( 5636349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer getContext().getFloatTypeSemantics(E->getArg(0)->getType())); 5646349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer Value *IsNormal = 5656349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer Builder.CreateFCmpUGE(Abs, ConstantFP::get(V->getContext(), Smallest), 5666349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer "isnormal"); 5676349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer V = Builder.CreateAnd(Eq, IsLessThanInf, "and"); 5686349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer V = Builder.CreateAnd(V, IsNormal, "and"); 5696349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); 5706349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer } 5716349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer 572ed074150c9a775c5e2e1c4ececeba18ba880ce7dChris Lattner case Builtin::BI__builtin_isfinite: { 573ef004ec5adb0e11815cef3435fa5ac7366d783a9Julien Lerouge // isfinite(x) --> x == x && fabs(x) != infinity; 574ed074150c9a775c5e2e1c4ececeba18ba880ce7dChris Lattner Value *V = EmitScalarExpr(E->getArg(0)); 575ed074150c9a775c5e2e1c4ececeba18ba880ce7dChris Lattner Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq"); 576258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 577176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *Abs = EmitFAbs(*this, V); 578ed074150c9a775c5e2e1c4ececeba18ba880ce7dChris Lattner Value *IsNotInf = 579ed074150c9a775c5e2e1c4ececeba18ba880ce7dChris Lattner Builder.CreateFCmpUNE(Abs, ConstantFP::getInfinity(V->getType()),"isinf"); 580258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 581ed074150c9a775c5e2e1c4ececeba18ba880ce7dChris Lattner V = Builder.CreateAnd(Eq, IsNotInf, "and"); 582ed074150c9a775c5e2e1c4ececeba18ba880ce7dChris Lattner return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); 583ed074150c9a775c5e2e1c4ececeba18ba880ce7dChris Lattner } 5847867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer 5857867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer case Builtin::BI__builtin_fpclassify: { 5867867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *V = EmitScalarExpr(E->getArg(5)); 5872acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *Ty = ConvertType(E->getArg(5)->getType()); 5887867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer 5897867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer // Create Result 5907867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer BasicBlock *Begin = Builder.GetInsertBlock(); 5917867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer BasicBlock *End = createBasicBlock("fpclassify_end", this->CurFn); 5927867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.SetInsertPoint(End); 5937867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer PHINode *Result = 594bbf3bacb3e0c1ebb3e8a4a8b1330404a7e379315Jay Foad Builder.CreatePHI(ConvertType(E->getArg(0)->getType()), 4, 5957867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer "fpclassify_result"); 5967867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer 5977867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer // if (V==0) return FP_ZERO 5987867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.SetInsertPoint(Begin); 5997867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *IsZero = Builder.CreateFCmpOEQ(V, Constant::getNullValue(Ty), 6007867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer "iszero"); 6017867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *ZeroLiteral = EmitScalarExpr(E->getArg(4)); 6027867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer BasicBlock *NotZero = createBasicBlock("fpclassify_not_zero", this->CurFn); 6037867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.CreateCondBr(IsZero, End, NotZero); 6047867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Result->addIncoming(ZeroLiteral, Begin); 6057867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer 6067867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer // if (V != V) return FP_NAN 6077867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.SetInsertPoint(NotZero); 6087867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *IsNan = Builder.CreateFCmpUNO(V, V, "cmp"); 6097867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *NanLiteral = EmitScalarExpr(E->getArg(0)); 6107867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer BasicBlock *NotNan = createBasicBlock("fpclassify_not_nan", this->CurFn); 6117867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.CreateCondBr(IsNan, End, NotNan); 6127867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Result->addIncoming(NanLiteral, NotZero); 6137867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer 6147867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer // if (fabs(V) == infinity) return FP_INFINITY 6157867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.SetInsertPoint(NotNan); 616176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *VAbs = EmitFAbs(*this, V); 6177867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *IsInf = 6187867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(V->getType()), 6197867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer "isinf"); 6207867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *InfLiteral = EmitScalarExpr(E->getArg(1)); 6217867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer BasicBlock *NotInf = createBasicBlock("fpclassify_not_inf", this->CurFn); 6227867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.CreateCondBr(IsInf, End, NotInf); 6237867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Result->addIncoming(InfLiteral, NotNan); 6247867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer 6257867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer // if (fabs(V) >= MIN_NORMAL) return FP_NORMAL else FP_SUBNORMAL 6267867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.SetInsertPoint(NotInf); 6277867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer APFloat Smallest = APFloat::getSmallestNormalized( 6287867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer getContext().getFloatTypeSemantics(E->getArg(5)->getType())); 6297867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *IsNormal = 6307867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.CreateFCmpUGE(VAbs, ConstantFP::get(V->getContext(), Smallest), 6317867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer "isnormal"); 6327867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *NormalResult = 6337867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.CreateSelect(IsNormal, EmitScalarExpr(E->getArg(2)), 6347867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer EmitScalarExpr(E->getArg(3))); 6357867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.CreateBr(End); 6367867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Result->addIncoming(NormalResult, NotInf); 6377867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer 6387867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer // return Result 6397867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.SetInsertPoint(End); 6407867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer return RValue::get(Result); 6417867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer } 642258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 643b52fe9ce99970955a5f581f5c66fcd89be9a268bEli Friedman case Builtin::BIalloca: 6446526de425dbfd368407566ff123b5bdc0924859aReid Kleckner case Builtin::BI_alloca: 6459e800e3dd80d77f6c47054738177bf824089f55aChris Lattner case Builtin::BI__builtin_alloca: { 6469e800e3dd80d77f6c47054738177bf824089f55aChris Lattner Value *Size = EmitScalarExpr(E->getArg(0)); 647578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer return RValue::get(Builder.CreateAlloca(Builder.getInt8Ty(), Size)); 6481caae959017b355e9bb61250d5a0d04edbf468b0Daniel Dunbar } 649e6dddfd907f6ea58daed5e26eeaacd893d98db9bEli Friedman case Builtin::BIbzero: 6501caae959017b355e9bb61250d5a0d04edbf468b0Daniel Dunbar case Builtin::BI__builtin_bzero: { 651ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman std::pair<llvm::Value*, unsigned> Dest = 652ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman EmitPointerWithAlignment(E->getArg(0)); 6533ecd785aff34381f3704d9cb28fe3ef85af759deMon P Wang Value *SizeVal = EmitScalarExpr(E->getArg(1)); 654ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman Builder.CreateMemSet(Dest.first, Builder.getInt8(0), SizeVal, 655ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman Dest.second, false); 656ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman return RValue::get(Dest.first); 6579e800e3dd80d77f6c47054738177bf824089f55aChris Lattner } 658e6ec205d6d0f4aec27bf49ca1e8fbb139acc2f2bEli Friedman case Builtin::BImemcpy: 659d4b32e46517358f34e8cfbea35010adfcc3786e0Eli Friedman case Builtin::BI__builtin_memcpy: { 660ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman std::pair<llvm::Value*, unsigned> Dest = 661ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman EmitPointerWithAlignment(E->getArg(0)); 662ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman std::pair<llvm::Value*, unsigned> Src = 663ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman EmitPointerWithAlignment(E->getArg(1)); 6643ecd785aff34381f3704d9cb28fe3ef85af759deMon P Wang Value *SizeVal = EmitScalarExpr(E->getArg(2)); 665ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman unsigned Align = std::min(Dest.second, Src.second); 666ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman Builder.CreateMemCpy(Dest.first, Src.first, SizeVal, Align, false); 667ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman return RValue::get(Dest.first); 6681caae959017b355e9bb61250d5a0d04edbf468b0Daniel Dunbar } 669258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 670a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner case Builtin::BI__builtin___memcpy_chk: { 671f3477c13eeaf11b32a41f181398fb5deffd0dd73Sylvestre Ledru // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memcpy iff cst1<=cst2. 672a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith llvm::APSInt Size, DstSize; 673a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) || 674a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext())) 675a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner break; 676a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner if (Size.ugt(DstSize)) 677a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner break; 678ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman std::pair<llvm::Value*, unsigned> Dest = 679ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman EmitPointerWithAlignment(E->getArg(0)); 680ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman std::pair<llvm::Value*, unsigned> Src = 681ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman EmitPointerWithAlignment(E->getArg(1)); 682a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); 683ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman unsigned Align = std::min(Dest.second, Src.second); 684ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman Builder.CreateMemCpy(Dest.first, Src.first, SizeVal, Align, false); 685ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman return RValue::get(Dest.first); 686a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner } 687258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 6888e2eab27056a78bf1db50ee09929438ed5ea9d93Fariborz Jahanian case Builtin::BI__builtin_objc_memmove_collectable: { 68955bcace250e1ff366e4482714b344b8cbc8be5f3Fariborz Jahanian Value *Address = EmitScalarExpr(E->getArg(0)); 69055bcace250e1ff366e4482714b344b8cbc8be5f3Fariborz Jahanian Value *SrcAddr = EmitScalarExpr(E->getArg(1)); 69155bcace250e1ff366e4482714b344b8cbc8be5f3Fariborz Jahanian Value *SizeVal = EmitScalarExpr(E->getArg(2)); 692258f930227c1a102c9c22eee88df65f748863425Jim Grosbach CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, 69355bcace250e1ff366e4482714b344b8cbc8be5f3Fariborz Jahanian Address, SrcAddr, SizeVal); 69455bcace250e1ff366e4482714b344b8cbc8be5f3Fariborz Jahanian return RValue::get(Address); 69555bcace250e1ff366e4482714b344b8cbc8be5f3Fariborz Jahanian } 696a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner 697a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner case Builtin::BI__builtin___memmove_chk: { 698f3477c13eeaf11b32a41f181398fb5deffd0dd73Sylvestre Ledru // fold __builtin_memmove_chk(x, y, cst1, cst2) to memmove iff cst1<=cst2. 699a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith llvm::APSInt Size, DstSize; 700a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) || 701a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext())) 702a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner break; 703a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner if (Size.ugt(DstSize)) 704a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner break; 705ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman std::pair<llvm::Value*, unsigned> Dest = 706ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman EmitPointerWithAlignment(E->getArg(0)); 707ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman std::pair<llvm::Value*, unsigned> Src = 708ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman EmitPointerWithAlignment(E->getArg(1)); 709a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); 710ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman unsigned Align = std::min(Dest.second, Src.second); 711ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman Builder.CreateMemMove(Dest.first, Src.first, SizeVal, Align, false); 712ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman return RValue::get(Dest.first); 713a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner } 714a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner 715e6ec205d6d0f4aec27bf49ca1e8fbb139acc2f2bEli Friedman case Builtin::BImemmove: 7161caae959017b355e9bb61250d5a0d04edbf468b0Daniel Dunbar case Builtin::BI__builtin_memmove: { 717ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman std::pair<llvm::Value*, unsigned> Dest = 718ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman EmitPointerWithAlignment(E->getArg(0)); 719ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman std::pair<llvm::Value*, unsigned> Src = 720ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman EmitPointerWithAlignment(E->getArg(1)); 7213ecd785aff34381f3704d9cb28fe3ef85af759deMon P Wang Value *SizeVal = EmitScalarExpr(E->getArg(2)); 722ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman unsigned Align = std::min(Dest.second, Src.second); 723ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman Builder.CreateMemMove(Dest.first, Src.first, SizeVal, Align, false); 724ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman return RValue::get(Dest.first); 7251caae959017b355e9bb61250d5a0d04edbf468b0Daniel Dunbar } 726e6ec205d6d0f4aec27bf49ca1e8fbb139acc2f2bEli Friedman case Builtin::BImemset: 7271caae959017b355e9bb61250d5a0d04edbf468b0Daniel Dunbar case Builtin::BI__builtin_memset: { 728ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman std::pair<llvm::Value*, unsigned> Dest = 729ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman EmitPointerWithAlignment(E->getArg(0)); 7309f0c7cc36d29cf591c33962931f5862847145f3eBenjamin Kramer Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), 7319f0c7cc36d29cf591c33962931f5862847145f3eBenjamin Kramer Builder.getInt8Ty()); 7323ecd785aff34381f3704d9cb28fe3ef85af759deMon P Wang Value *SizeVal = EmitScalarExpr(E->getArg(2)); 733ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman Builder.CreateMemSet(Dest.first, ByteVal, SizeVal, Dest.second, false); 734ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman return RValue::get(Dest.first); 735d4b32e46517358f34e8cfbea35010adfcc3786e0Eli Friedman } 736a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner case Builtin::BI__builtin___memset_chk: { 737f3477c13eeaf11b32a41f181398fb5deffd0dd73Sylvestre Ledru // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2. 738a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith llvm::APSInt Size, DstSize; 739a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) || 740a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext())) 741a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner break; 742a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner if (Size.ugt(DstSize)) 743a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner break; 744ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman std::pair<llvm::Value*, unsigned> Dest = 745ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman EmitPointerWithAlignment(E->getArg(0)); 746a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), 747a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner Builder.getInt8Ty()); 748a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); 749ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman Builder.CreateMemSet(Dest.first, ByteVal, SizeVal, Dest.second, false); 750ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman return RValue::get(Dest.first); 751a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner } 752fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall case Builtin::BI__builtin_dwarf_cfa: { 753fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall // The offset in bytes from the first argument to the CFA. 754fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall // 755fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall // Why on earth is this in the frontend? Is there any reason at 756fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall // all that the backend can't reasonably determine this while 757fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall // lowering llvm.eh.dwarf.cfa()? 758fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall // 759fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall // TODO: If there's a satisfactory reason, add a target hook for 760fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall // this instead of hard-coding 0, which is correct for most targets. 761fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall int32_t Offset = 0; 762fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall 7638dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa); 764258f930227c1a102c9c22eee88df65f748863425Jim Grosbach return RValue::get(Builder.CreateCall(F, 76577b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner llvm::ConstantInt::get(Int32Ty, Offset))); 766fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall } 767256f77e431bc6b920ec94cf0bb4ad339ca21b8c9Eli Friedman case Builtin::BI__builtin_return_address: { 76883c2a98012a65b51be66fd76c3a1b13ed782c558Anton Korobeynikov Value *Depth = EmitScalarExpr(E->getArg(0)); 769578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer Depth = Builder.CreateIntCast(Depth, Int32Ty, false); 7708dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::returnaddress); 77183c2a98012a65b51be66fd76c3a1b13ed782c558Anton Korobeynikov return RValue::get(Builder.CreateCall(F, Depth)); 772256f77e431bc6b920ec94cf0bb4ad339ca21b8c9Eli Friedman } 773256f77e431bc6b920ec94cf0bb4ad339ca21b8c9Eli Friedman case Builtin::BI__builtin_frame_address: { 77483c2a98012a65b51be66fd76c3a1b13ed782c558Anton Korobeynikov Value *Depth = EmitScalarExpr(E->getArg(0)); 775578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer Depth = Builder.CreateIntCast(Depth, Int32Ty, false); 7768dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::frameaddress); 77783c2a98012a65b51be66fd76c3a1b13ed782c558Anton Korobeynikov return RValue::get(Builder.CreateCall(F, Depth)); 778256f77e431bc6b920ec94cf0bb4ad339ca21b8c9Eli Friedman } 7793b660efb9f9fa3e87096f4a96a2093cd17c43c2eEli Friedman case Builtin::BI__builtin_extract_return_addr: { 780492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall Value *Address = EmitScalarExpr(E->getArg(0)); 781492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall Value *Result = getTargetHooks().decodeReturnAddress(*this, Address); 782492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall return RValue::get(Result); 783492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall } 784492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall case Builtin::BI__builtin_frob_return_addr: { 785492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall Value *Address = EmitScalarExpr(E->getArg(0)); 786492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall Value *Result = getTargetHooks().encodeReturnAddress(*this, Address); 787492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall return RValue::get(Result); 7883b660efb9f9fa3e87096f4a96a2093cd17c43c2eEli Friedman } 7896374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall case Builtin::BI__builtin_dwarf_sp_column: { 7902acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::IntegerType *Ty 7916374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall = cast<llvm::IntegerType>(ConvertType(E->getType())); 7926374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall int Column = getTargetHooks().getDwarfEHStackPointer(CGM); 7936374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall if (Column == -1) { 7946374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall CGM.ErrorUnsupported(E, "__builtin_dwarf_sp_column"); 7956374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall return RValue::get(llvm::UndefValue::get(Ty)); 7966374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall } 7976374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall return RValue::get(llvm::ConstantInt::get(Ty, Column, true)); 7986374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall } 7996374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall case Builtin::BI__builtin_init_dwarf_reg_size_table: { 8006374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall Value *Address = EmitScalarExpr(E->getArg(0)); 8016374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall if (getTargetHooks().initDwarfEHRegSizeTable(*this, Address)) 8026374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall CGM.ErrorUnsupported(E, "__builtin_init_dwarf_reg_size_table"); 8036374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall return RValue::get(llvm::UndefValue::get(ConvertType(E->getType()))); 8046374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall } 8057ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall case Builtin::BI__builtin_eh_return: { 8067ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall Value *Int = EmitScalarExpr(E->getArg(0)); 8077ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall Value *Ptr = EmitScalarExpr(E->getArg(1)); 8087ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall 8092acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType()); 8107ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) && 8117ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall "LLVM's __builtin_eh_return only supports 32- and 64-bit variants"); 8127ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall Value *F = CGM.getIntrinsic(IntTy->getBitWidth() == 32 8137ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall ? Intrinsic::eh_return_i32 8148dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer : Intrinsic::eh_return_i64); 8157ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall Builder.CreateCall2(F, Int, Ptr); 816cd5b22e12b6513163dd131589746c194090f14e6John McCall Builder.CreateUnreachable(); 817cd5b22e12b6513163dd131589746c194090f14e6John McCall 818cd5b22e12b6513163dd131589746c194090f14e6John McCall // We do need to preserve an insertion point. 819d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall EmitBlock(createBasicBlock("builtin_eh_return.cont")); 820cd5b22e12b6513163dd131589746c194090f14e6John McCall 8216bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 8227ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall } 823a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman case Builtin::BI__builtin_unwind_init: { 8248dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init); 825a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman return RValue::get(Builder.CreateCall(F)); 826a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman } 8275e11085830d4d4c53ff75575ab75889ee5126854John McCall case Builtin::BI__builtin_extend_pointer: { 8285e11085830d4d4c53ff75575ab75889ee5126854John McCall // Extends a pointer to the size of an _Unwind_Word, which is 829d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // uint64_t on all platforms. Generally this gets poked into a 830d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // register and eventually used as an address, so if the 831d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // addressing registers are wider than pointers and the platform 832d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // doesn't implicitly ignore high-order bits when doing 833d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // addressing, we need to make sure we zext / sext based on 834d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // the platform's expectations. 8355e11085830d4d4c53ff75575ab75889ee5126854John McCall // 8365e11085830d4d4c53ff75575ab75889ee5126854John McCall // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html 837d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall 838d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // Cast the pointer to intptr_t. 8395e11085830d4d4c53ff75575ab75889ee5126854John McCall Value *Ptr = EmitScalarExpr(E->getArg(0)); 840d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall Value *Result = Builder.CreatePtrToInt(Ptr, IntPtrTy, "extend.cast"); 841d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall 842d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // If that's 64 bits, we're done. 843d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall if (IntPtrTy->getBitWidth() == 64) 844d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall return RValue::get(Result); 845d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall 846d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // Otherwise, ask the codegen data what to do. 847492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall if (getTargetHooks().extendPointerWithSExt()) 848d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall return RValue::get(Builder.CreateSExt(Result, Int64Ty, "extend.sext")); 849d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall else 850d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall return RValue::get(Builder.CreateZExt(Result, Int64Ty, "extend.zext")); 8515e11085830d4d4c53ff75575ab75889ee5126854John McCall } 852a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman case Builtin::BI__builtin_setjmp: { 85378673d9f910e8dfe13248c2426c51d8f9fb28572John McCall // Buffer is a void**. 854a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman Value *Buf = EmitScalarExpr(E->getArg(0)); 85578673d9f910e8dfe13248c2426c51d8f9fb28572John McCall 85678673d9f910e8dfe13248c2426c51d8f9fb28572John McCall // Store the frame pointer to the setjmp buffer. 857a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman Value *FrameAddr = 85878673d9f910e8dfe13248c2426c51d8f9fb28572John McCall Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress), 85977b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner ConstantInt::get(Int32Ty, 0)); 860a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman Builder.CreateStore(FrameAddr, Buf); 86178673d9f910e8dfe13248c2426c51d8f9fb28572John McCall 8626d172e2985346e55095c75f456901ea5d40fddaaJim Grosbach // Store the stack pointer to the setjmp buffer. 8636d172e2985346e55095c75f456901ea5d40fddaaJim Grosbach Value *StackAddr = 8646d172e2985346e55095c75f456901ea5d40fddaaJim Grosbach Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave)); 8656d172e2985346e55095c75f456901ea5d40fddaaJim Grosbach Value *StackSaveSlot = 86677b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner Builder.CreateGEP(Buf, ConstantInt::get(Int32Ty, 2)); 8676d172e2985346e55095c75f456901ea5d40fddaaJim Grosbach Builder.CreateStore(StackAddr, StackSaveSlot); 8686d172e2985346e55095c75f456901ea5d40fddaaJim Grosbach 86978673d9f910e8dfe13248c2426c51d8f9fb28572John McCall // Call LLVM's EH setjmp, which is lightweight. 87078673d9f910e8dfe13248c2426c51d8f9fb28572John McCall Value *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp); 871d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Buf = Builder.CreateBitCast(Buf, Int8PtrTy); 872a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman return RValue::get(Builder.CreateCall(F, Buf)); 873a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman } 874a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman case Builtin::BI__builtin_longjmp: { 875a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman Value *Buf = EmitScalarExpr(E->getArg(0)); 876d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Buf = Builder.CreateBitCast(Buf, Int8PtrTy); 87778673d9f910e8dfe13248c2426c51d8f9fb28572John McCall 87878673d9f910e8dfe13248c2426c51d8f9fb28572John McCall // Call LLVM's EH longjmp, which is lightweight. 87978673d9f910e8dfe13248c2426c51d8f9fb28572John McCall Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf); 88078673d9f910e8dfe13248c2426c51d8f9fb28572John McCall 881cd5b22e12b6513163dd131589746c194090f14e6John McCall // longjmp doesn't return; mark this as unreachable. 882cd5b22e12b6513163dd131589746c194090f14e6John McCall Builder.CreateUnreachable(); 883cd5b22e12b6513163dd131589746c194090f14e6John McCall 884cd5b22e12b6513163dd131589746c194090f14e6John McCall // We do need to preserve an insertion point. 885d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall EmitBlock(createBasicBlock("longjmp.cont")); 886cd5b22e12b6513163dd131589746c194090f14e6John McCall 8876bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 888a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman } 8891ffe281890f3cd7728316b45a1f3dd4d3120af7bMon P Wang case Builtin::BI__sync_fetch_and_add: 8901ffe281890f3cd7728316b45a1f3dd4d3120af7bMon P Wang case Builtin::BI__sync_fetch_and_sub: 8915caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_or: 8925caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_and: 8935caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_xor: 894176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__sync_fetch_and_nand: 8955caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_add_and_fetch: 8965caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_sub_and_fetch: 8975caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_and_and_fetch: 8985caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_or_and_fetch: 8995caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_xor_and_fetch: 900176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__sync_nand_and_fetch: 9015caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_val_compare_and_swap: 9025caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_bool_compare_and_swap: 9035caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_test_and_set: 9045caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_release: 90523aa9c8ca0bc441aab2a38a4c9fc7a1c9e9ac16aChris Lattner case Builtin::BI__sync_swap: 906b219cfc4d75f0a03630b7c4509ef791b7e97b2c8David Blaikie llvm_unreachable("Shouldn't make it through sema"); 9075caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_add_1: 9085caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_add_2: 9095caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_add_4: 9105caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_add_8: 9115caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_add_16: 912c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Add, E); 9135caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_sub_1: 9145caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_sub_2: 9155caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_sub_4: 9165caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_sub_8: 9175caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_sub_16: 918c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Sub, E); 9195caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_or_1: 9205caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_or_2: 9215caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_or_4: 9225caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_or_8: 9235caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_or_16: 924c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Or, E); 9255caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_and_1: 9265caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_and_2: 9275caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_and_4: 9285caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_and_8: 9295caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_and_16: 930c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::And, E); 9315caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_xor_1: 9325caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_xor_2: 9335caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_xor_4: 9345caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_xor_8: 9355caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_xor_16: 936c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xor, E); 937176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__sync_fetch_and_nand_1: 938176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__sync_fetch_and_nand_2: 939176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__sync_fetch_and_nand_4: 940176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__sync_fetch_and_nand_8: 941176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__sync_fetch_and_nand_16: 942176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Nand, E); 9431eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 9445caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner // Clang extensions: not overloaded yet. 9451ffe281890f3cd7728316b45a1f3dd4d3120af7bMon P Wang case Builtin::BI__sync_fetch_and_min: 946c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Min, E); 9471ffe281890f3cd7728316b45a1f3dd4d3120af7bMon P Wang case Builtin::BI__sync_fetch_and_max: 948c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Max, E); 9491ffe281890f3cd7728316b45a1f3dd4d3120af7bMon P Wang case Builtin::BI__sync_fetch_and_umin: 950c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMin, E); 9511ffe281890f3cd7728316b45a1f3dd4d3120af7bMon P Wang case Builtin::BI__sync_fetch_and_umax: 952c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMax, E); 9530002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar 9545caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_add_and_fetch_1: 9555caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_add_and_fetch_2: 9565caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_add_and_fetch_4: 9575caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_add_and_fetch_8: 9585caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_add_and_fetch_16: 959c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Add, E, 9600002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar llvm::Instruction::Add); 9615caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_sub_and_fetch_1: 9625caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_sub_and_fetch_2: 9635caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_sub_and_fetch_4: 9645caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_sub_and_fetch_8: 9655caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_sub_and_fetch_16: 966c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Sub, E, 9670002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar llvm::Instruction::Sub); 9685caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_and_and_fetch_1: 9695caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_and_and_fetch_2: 9705caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_and_and_fetch_4: 9715caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_and_and_fetch_8: 9725caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_and_and_fetch_16: 973c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::And, E, 9740002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar llvm::Instruction::And); 9755caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_or_and_fetch_1: 9765caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_or_and_fetch_2: 9775caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_or_and_fetch_4: 9785caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_or_and_fetch_8: 9795caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_or_and_fetch_16: 980c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Or, E, 9810002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar llvm::Instruction::Or); 9825caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_xor_and_fetch_1: 9835caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_xor_and_fetch_2: 9845caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_xor_and_fetch_4: 9855caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_xor_and_fetch_8: 9865caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_xor_and_fetch_16: 987c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Xor, E, 9880002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar llvm::Instruction::Xor); 989176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__sync_nand_and_fetch_1: 990176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__sync_nand_and_fetch_2: 991176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__sync_nand_and_fetch_4: 992176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__sync_nand_and_fetch_8: 993176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__sync_nand_and_fetch_16: 994176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Nand, E, 995176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::Instruction::And, true); 9961eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 9975caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_val_compare_and_swap_1: 9985caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_val_compare_and_swap_2: 9995caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_val_compare_and_swap_4: 10005caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_val_compare_and_swap_8: 1001cb61a7bbe635cfa941b1aeaaa1fbda1bf900ee51Daniel Dunbar case Builtin::BI__sync_val_compare_and_swap_16: { 100226815d97c5743481e317f17a8d53a6819d061862John McCall QualType T = E->getType(); 1003d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall llvm::Value *DestPtr = EmitScalarExpr(E->getArg(0)); 1004956a5a17713deb1b5b27893303c4f308a1bd2a62Micah Villmow unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); 1005258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 10069cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::IntegerType *IntType = 1007d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall llvm::IntegerType::get(getLLVMContext(), 1008d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall getContext().getTypeSize(T)); 10099cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); 1010db4325b098eff5e9e660db19f0148423fb21f27fChandler Carruth 101126815d97c5743481e317f17a8d53a6819d061862John McCall Value *Args[3]; 101226815d97c5743481e317f17a8d53a6819d061862John McCall Args[0] = Builder.CreateBitCast(DestPtr, IntPtrType); 1013d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Args[1] = EmitScalarExpr(E->getArg(1)); 10142acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ValueType = Args[1]->getType(); 1015d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Args[1] = EmitToInt(*this, Args[1], T, IntType); 1016d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Args[2] = EmitToInt(*this, EmitScalarExpr(E->getArg(2)), T, IntType); 101726815d97c5743481e317f17a8d53a6819d061862John McCall 1018c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman Value *Result = Builder.CreateAtomicCmpXchg(Args[0], Args[1], Args[2], 1019651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::SequentiallyConsistent, 1020c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman llvm::SequentiallyConsistent); 1021c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Result = Builder.CreateExtractValue(Result, 0); 1022d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Result = EmitFromInt(*this, Result, T, ValueType); 102326815d97c5743481e317f17a8d53a6819d061862John McCall return RValue::get(Result); 1024022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson } 10250002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar 10265caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_bool_compare_and_swap_1: 10275caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_bool_compare_and_swap_2: 10285caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_bool_compare_and_swap_4: 10295caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_bool_compare_and_swap_8: 1030cb61a7bbe635cfa941b1aeaaa1fbda1bf900ee51Daniel Dunbar case Builtin::BI__sync_bool_compare_and_swap_16: { 103126815d97c5743481e317f17a8d53a6819d061862John McCall QualType T = E->getArg(1)->getType(); 1032d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall llvm::Value *DestPtr = EmitScalarExpr(E->getArg(0)); 1033956a5a17713deb1b5b27893303c4f308a1bd2a62Micah Villmow unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); 1034258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 10359cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::IntegerType *IntType = 1036d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall llvm::IntegerType::get(getLLVMContext(), 1037d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall getContext().getTypeSize(T)); 10389cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); 1039db4325b098eff5e9e660db19f0148423fb21f27fChandler Carruth 104026815d97c5743481e317f17a8d53a6819d061862John McCall Value *Args[3]; 104126815d97c5743481e317f17a8d53a6819d061862John McCall Args[0] = Builder.CreateBitCast(DestPtr, IntPtrType); 1042d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Args[1] = EmitToInt(*this, EmitScalarExpr(E->getArg(1)), T, IntType); 1043d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Args[2] = EmitToInt(*this, EmitScalarExpr(E->getArg(2)), T, IntType); 104426815d97c5743481e317f17a8d53a6819d061862John McCall 1045c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Value *Pair = Builder.CreateAtomicCmpXchg(Args[0], Args[1], Args[2], 1046c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::SequentiallyConsistent, 1047c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::SequentiallyConsistent); 1048c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Value *Result = Builder.CreateExtractValue(Pair, 1); 10490002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar // zext bool to int. 105026815d97c5743481e317f17a8d53a6819d061862John McCall Result = Builder.CreateZExt(Result, ConvertType(E->getType())); 105126815d97c5743481e317f17a8d53a6819d061862John McCall return RValue::get(Result); 10520002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar } 10530002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar 105423aa9c8ca0bc441aab2a38a4c9fc7a1c9e9ac16aChris Lattner case Builtin::BI__sync_swap_1: 105523aa9c8ca0bc441aab2a38a4c9fc7a1c9e9ac16aChris Lattner case Builtin::BI__sync_swap_2: 105623aa9c8ca0bc441aab2a38a4c9fc7a1c9e9ac16aChris Lattner case Builtin::BI__sync_swap_4: 105723aa9c8ca0bc441aab2a38a4c9fc7a1c9e9ac16aChris Lattner case Builtin::BI__sync_swap_8: 105823aa9c8ca0bc441aab2a38a4c9fc7a1c9e9ac16aChris Lattner case Builtin::BI__sync_swap_16: 1059c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E); 106023aa9c8ca0bc441aab2a38a4c9fc7a1c9e9ac16aChris Lattner 10615caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_test_and_set_1: 10625caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_test_and_set_2: 10635caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_test_and_set_4: 10645caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_test_and_set_8: 10655caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_test_and_set_16: 1066c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E); 1067cb61a7bbe635cfa941b1aeaaa1fbda1bf900ee51Daniel Dunbar 10685caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_release_1: 10695caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_release_2: 10705caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_release_4: 10715caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_release_8: 1072f58cd9bca9c2e7373300fc8bb7c57cff7e4eda4fChris Lattner case Builtin::BI__sync_lock_release_16: { 1073f58cd9bca9c2e7373300fc8bb7c57cff7e4eda4fChris Lattner Value *Ptr = EmitScalarExpr(E->getArg(0)); 1074eb43f4a8f133c2bc510ae136a556e92b68a6ff44Eli Friedman QualType ElTy = E->getArg(0)->getType()->getPointeeType(); 1075eb43f4a8f133c2bc510ae136a556e92b68a6ff44Eli Friedman CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy); 1076ff993202abf6f5dc41c584c7103f5d39f248b3ddEli Friedman llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(), 1077ff993202abf6f5dc41c584c7103f5d39f248b3ddEli Friedman StoreSize.getQuantity() * 8); 1078ff993202abf6f5dc41c584c7103f5d39f248b3ddEli Friedman Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo()); 1079258f930227c1a102c9c22eee88df65f748863425Jim Grosbach llvm::StoreInst *Store = 1080ff993202abf6f5dc41c584c7103f5d39f248b3ddEli Friedman Builder.CreateStore(llvm::Constant::getNullValue(ITy), Ptr); 1081eb43f4a8f133c2bc510ae136a556e92b68a6ff44Eli Friedman Store->setAlignment(StoreSize.getQuantity()); 1082eb43f4a8f133c2bc510ae136a556e92b68a6ff44Eli Friedman Store->setAtomic(llvm::Release); 10836bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 1084f58cd9bca9c2e7373300fc8bb7c57cff7e4eda4fChris Lattner } 1085ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar 1086f58cd9bca9c2e7373300fc8bb7c57cff7e4eda4fChris Lattner case Builtin::BI__sync_synchronize: { 1087c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman // We assume this is supposed to correspond to a C++0x-style 1088c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman // sequentially-consistent fence (i.e. this is only usable for 1089c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman // synchonization, not device I/O or anything like that). This intrinsic 1090258f930227c1a102c9c22eee88df65f748863425Jim Grosbach // is really badly designed in the sense that in theory, there isn't 1091c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman // any way to safely use it... but in practice, it mostly works 1092c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman // to use it with non-atomic loads and stores to get acquire/release 1093c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman // semantics. 1094c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman Builder.CreateFence(llvm::SequentiallyConsistent); 10956bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 1096f58cd9bca9c2e7373300fc8bb7c57cff7e4eda4fChris Lattner } 10971eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 10982c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case Builtin::BI__c11_atomic_is_lock_free: 10992c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case Builtin::BI__atomic_is_lock_free: { 11002c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith // Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the 11012c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith // __c11 builtin, ptr is 0 (indicating a properly-aligned object), since 11022c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith // _Atomic(T) is always properly-aligned. 11032c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith const char *LibCallName = "__atomic_is_lock_free"; 11042c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith CallArgList Args; 11052c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Args.add(RValue::get(EmitScalarExpr(E->getArg(0))), 11062c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith getContext().getSizeType()); 11072c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith if (BuiltinID == Builtin::BI__atomic_is_lock_free) 11082c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Args.add(RValue::get(EmitScalarExpr(E->getArg(1))), 11092c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith getContext().VoidPtrTy); 11102c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith else 11112c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Args.add(RValue::get(llvm::Constant::getNullValue(VoidPtrTy)), 11122c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith getContext().VoidPtrTy); 11132c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith const CGFunctionInfo &FuncInfo = 11140f3d0970dcdf6cf17550b86838dff12813968dbcJohn McCall CGM.getTypes().arrangeFreeFunctionCall(E->getType(), Args, 11150f3d0970dcdf6cf17550b86838dff12813968dbcJohn McCall FunctionType::ExtInfo(), 11160f3d0970dcdf6cf17550b86838dff12813968dbcJohn McCall RequiredArgs::All); 11172c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo); 11182c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName); 11192c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith return EmitCall(FuncInfo, Func, ReturnValueSlot(), Args); 11202c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 11212c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 11222c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case Builtin::BI__atomic_test_and_set: { 11232c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith // Look at the argument type to determine whether this is a volatile 11242c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith // operation. The parameter type is always volatile. 11252c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType(); 11262c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith bool Volatile = 11272c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified(); 11282c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 11292c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Value *Ptr = EmitScalarExpr(E->getArg(0)); 1130956a5a17713deb1b5b27893303c4f308a1bd2a62Micah Villmow unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace(); 11312c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace)); 11322c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Value *NewVal = Builder.getInt8(1); 11332c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Value *Order = EmitScalarExpr(E->getArg(1)); 11342c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith if (isa<llvm::ConstantInt>(Order)) { 11352c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 11366bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines AtomicRMWInst *Result = nullptr; 11372c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith switch (ord) { 11382c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case 0: // memory_order_relaxed 11392c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith default: // invalid order 11402c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, 11412c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Ptr, NewVal, 11422c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::Monotonic); 11432c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith break; 11442c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case 1: // memory_order_consume 11452c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case 2: // memory_order_acquire 11462c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, 11472c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Ptr, NewVal, 11482c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::Acquire); 11492c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith break; 11502c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case 3: // memory_order_release 11512c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, 11522c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Ptr, NewVal, 11532c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::Release); 11542c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith break; 11552c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case 4: // memory_order_acq_rel 11562c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, 11572c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Ptr, NewVal, 11582c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::AcquireRelease); 11592c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith break; 11602c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case 5: // memory_order_seq_cst 11612c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, 11622c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Ptr, NewVal, 11632c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::SequentiallyConsistent); 11642c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith break; 11652c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 11662c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Result->setVolatile(Volatile); 11672c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith return RValue::get(Builder.CreateIsNotNull(Result, "tobool")); 11682c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 11692c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 11702c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 11712c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 11722c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::BasicBlock *BBs[5] = { 11732c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith createBasicBlock("monotonic", CurFn), 11742c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith createBasicBlock("acquire", CurFn), 11752c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith createBasicBlock("release", CurFn), 11762c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith createBasicBlock("acqrel", CurFn), 11772c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith createBasicBlock("seqcst", CurFn) 11782c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith }; 11792c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::AtomicOrdering Orders[5] = { 11802c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::Monotonic, llvm::Acquire, llvm::Release, 11812c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::AcquireRelease, llvm::SequentiallyConsistent 11822c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith }; 11832c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 11842c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 11852c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]); 11862c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 11872c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Builder.SetInsertPoint(ContBB); 11882c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith PHINode *Result = Builder.CreatePHI(Int8Ty, 5, "was_set"); 11892c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 11902c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith for (unsigned i = 0; i < 5; ++i) { 11912c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Builder.SetInsertPoint(BBs[i]); 11922c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith AtomicRMWInst *RMW = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, 11932c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Ptr, NewVal, Orders[i]); 11942c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith RMW->setVolatile(Volatile); 11952c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Result->addIncoming(RMW, BBs[i]); 11962c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Builder.CreateBr(ContBB); 11972c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 11982c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 11992c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(0), BBs[0]); 12002c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(1), BBs[1]); 12012c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(2), BBs[1]); 12022c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(3), BBs[2]); 12032c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(4), BBs[3]); 12042c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(5), BBs[4]); 12052c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 12062c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Builder.SetInsertPoint(ContBB); 12072c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith return RValue::get(Builder.CreateIsNotNull(Result, "tobool")); 12082c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 12092c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 12102c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case Builtin::BI__atomic_clear: { 12112c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType(); 12122c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith bool Volatile = 12132c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified(); 12142c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 12152c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Value *Ptr = EmitScalarExpr(E->getArg(0)); 1216956a5a17713deb1b5b27893303c4f308a1bd2a62Micah Villmow unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace(); 12172c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace)); 12182c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Value *NewVal = Builder.getInt8(0); 12192c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Value *Order = EmitScalarExpr(E->getArg(1)); 12202c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith if (isa<llvm::ConstantInt>(Order)) { 12212c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 12222c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile); 12232c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Store->setAlignment(1); 12242c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith switch (ord) { 12252c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case 0: // memory_order_relaxed 12262c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith default: // invalid order 12272c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Store->setOrdering(llvm::Monotonic); 12282c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith break; 12292c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case 3: // memory_order_release 12302c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Store->setOrdering(llvm::Release); 12312c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith break; 12322c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case 5: // memory_order_seq_cst 12332c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Store->setOrdering(llvm::SequentiallyConsistent); 12342c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith break; 12352c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 12366bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 12372c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 12382c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 12392c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 12402c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 12412c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::BasicBlock *BBs[3] = { 12422c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith createBasicBlock("monotonic", CurFn), 12432c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith createBasicBlock("release", CurFn), 12442c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith createBasicBlock("seqcst", CurFn) 12452c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith }; 12462c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::AtomicOrdering Orders[3] = { 12472c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::Monotonic, llvm::Release, llvm::SequentiallyConsistent 12482c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith }; 12492c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 12502c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 12512c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]); 12522c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 12532c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith for (unsigned i = 0; i < 3; ++i) { 12542c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Builder.SetInsertPoint(BBs[i]); 12552c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile); 12562c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Store->setAlignment(1); 12572c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Store->setOrdering(Orders[i]); 12582c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Builder.CreateBr(ContBB); 12592c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 12602c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 12612c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(0), BBs[0]); 12622c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(3), BBs[1]); 12632c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(5), BBs[2]); 12642c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 12652c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Builder.SetInsertPoint(ContBB); 12666bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 12672c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 12682c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 1269276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman case Builtin::BI__atomic_thread_fence: 1270fafbf06732746f3ceca21d452d77b144ba8652aeRichard Smith case Builtin::BI__atomic_signal_fence: 1271fafbf06732746f3ceca21d452d77b144ba8652aeRichard Smith case Builtin::BI__c11_atomic_thread_fence: 1272fafbf06732746f3ceca21d452d77b144ba8652aeRichard Smith case Builtin::BI__c11_atomic_signal_fence: { 1273276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman llvm::SynchronizationScope Scope; 1274fafbf06732746f3ceca21d452d77b144ba8652aeRichard Smith if (BuiltinID == Builtin::BI__atomic_signal_fence || 1275fafbf06732746f3ceca21d452d77b144ba8652aeRichard Smith BuiltinID == Builtin::BI__c11_atomic_signal_fence) 1276276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Scope = llvm::SingleThread; 1277276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman else 1278276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Scope = llvm::CrossThread; 1279276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Value *Order = EmitScalarExpr(E->getArg(0)); 1280276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman if (isa<llvm::ConstantInt>(Order)) { 1281276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1282276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman switch (ord) { 1283276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman case 0: // memory_order_relaxed 1284276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman default: // invalid order 1285276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman break; 1286276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman case 1: // memory_order_consume 1287276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman case 2: // memory_order_acquire 1288276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateFence(llvm::Acquire, Scope); 1289276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman break; 1290276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman case 3: // memory_order_release 1291276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateFence(llvm::Release, Scope); 1292276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman break; 1293276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman case 4: // memory_order_acq_rel 1294276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateFence(llvm::AcquireRelease, Scope); 1295276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman break; 1296276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman case 5: // memory_order_seq_cst 1297276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateFence(llvm::SequentiallyConsistent, Scope); 1298276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman break; 1299276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman } 13006bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 1301276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman } 1302276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman 1303276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB; 1304276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman AcquireBB = createBasicBlock("acquire", CurFn); 1305276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman ReleaseBB = createBasicBlock("release", CurFn); 1306276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman AcqRelBB = createBasicBlock("acqrel", CurFn); 1307276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman SeqCstBB = createBasicBlock("seqcst", CurFn); 1308276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1309276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman 1310276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1311276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman llvm::SwitchInst *SI = Builder.CreateSwitch(Order, ContBB); 1312276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman 1313276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.SetInsertPoint(AcquireBB); 1314276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateFence(llvm::Acquire, Scope); 1315276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateBr(ContBB); 1316276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman SI->addCase(Builder.getInt32(1), AcquireBB); 1317276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman SI->addCase(Builder.getInt32(2), AcquireBB); 1318276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman 1319276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.SetInsertPoint(ReleaseBB); 1320276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateFence(llvm::Release, Scope); 1321276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateBr(ContBB); 1322276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman SI->addCase(Builder.getInt32(3), ReleaseBB); 1323276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman 1324276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.SetInsertPoint(AcqRelBB); 1325276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateFence(llvm::AcquireRelease, Scope); 1326276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateBr(ContBB); 1327276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman SI->addCase(Builder.getInt32(4), AcqRelBB); 1328276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman 1329276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.SetInsertPoint(SeqCstBB); 1330276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateFence(llvm::SequentiallyConsistent, Scope); 1331276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateBr(ContBB); 1332276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman SI->addCase(Builder.getInt32(5), SeqCstBB); 1333276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman 1334276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.SetInsertPoint(ContBB); 13356bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 1336276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman } 1337276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman 1338ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar // Library functions with special handling. 1339ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar case Builtin::BIsqrt: 1340ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar case Builtin::BIsqrtf: 1341ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar case Builtin::BIsqrtl: { 1342ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel // Transform a call to sqrt* into a @llvm.sqrt.* intrinsic call, but only 1343ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel // in finite- or unsafe-math mode (the intrinsic has different semantics 1344ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel // for handling negative numbers compared to the library function, so 1345ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel // -fmath-errno=0 is not enough). 1346ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel if (!FD->hasAttr<ConstAttr>()) 1347ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel break; 1348ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel if (!(CGM.getCodeGenOpts().UnsafeFPMath || 1349ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel CGM.getCodeGenOpts().NoNaNsFPMath)) 1350ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel break; 1351ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel Value *Arg0 = EmitScalarExpr(E->getArg(0)); 1352ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel llvm::Type *ArgType = Arg0->getType(); 1353ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel Value *F = CGM.getIntrinsic(Intrinsic::sqrt, ArgType); 1354ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel return RValue::get(Builder.CreateCall(F, Arg0)); 1355ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar } 1356ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar 1357ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar case Builtin::BIpow: 1358ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar case Builtin::BIpowf: 1359ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar case Builtin::BIpowl: { 13600323a7887b26f727080d2aa38f1a325dd23a1ee9Eli Bendersky // Transform a call to pow* into a @llvm.pow.* intrinsic call. 13610323a7887b26f727080d2aa38f1a325dd23a1ee9Eli Bendersky if (!FD->hasAttr<ConstAttr>()) 13620323a7887b26f727080d2aa38f1a325dd23a1ee9Eli Bendersky break; 13630323a7887b26f727080d2aa38f1a325dd23a1ee9Eli Bendersky Value *Base = EmitScalarExpr(E->getArg(0)); 13640323a7887b26f727080d2aa38f1a325dd23a1ee9Eli Bendersky Value *Exponent = EmitScalarExpr(E->getArg(1)); 13650323a7887b26f727080d2aa38f1a325dd23a1ee9Eli Bendersky llvm::Type *ArgType = Base->getType(); 13660323a7887b26f727080d2aa38f1a325dd23a1ee9Eli Bendersky Value *F = CGM.getIntrinsic(Intrinsic::pow, ArgType); 13670323a7887b26f727080d2aa38f1a325dd23a1ee9Eli Bendersky return RValue::get(Builder.CreateCall2(F, Base, Exponent)); 1368ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar } 1369ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman 1370094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich case Builtin::BIfma: 1371094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich case Builtin::BIfmaf: 1372094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich case Builtin::BIfmal: 1373094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich case Builtin::BI__builtin_fma: 1374094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich case Builtin::BI__builtin_fmaf: 1375094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich case Builtin::BI__builtin_fmal: { 1376094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich // Rewrite fma to intrinsic. 1377094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich Value *FirstArg = EmitScalarExpr(E->getArg(0)); 13789cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *ArgType = FirstArg->getType(); 13798dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::fma, ArgType); 1380094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich return RValue::get(Builder.CreateCall3(F, FirstArg, 1381094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich EmitScalarExpr(E->getArg(1)), 1382578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer EmitScalarExpr(E->getArg(2)))); 1383094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich } 1384094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich 1385ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman case Builtin::BI__builtin_signbit: 1386ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman case Builtin::BI__builtin_signbitf: 1387ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman case Builtin::BI__builtin_signbitl: { 1388ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman LLVMContext &C = CGM.getLLVMContext(); 1389ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman 1390ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman Value *Arg = EmitScalarExpr(E->getArg(0)); 13912acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ArgTy = Arg->getType(); 1392ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman int ArgWidth = ArgTy->getPrimitiveSizeInBits(); 13932acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ArgIntTy = llvm::IntegerType::get(C, ArgWidth); 1394ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman Value *BCArg = Builder.CreateBitCast(Arg, ArgIntTy); 1395176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (ArgTy->isPPC_FP128Ty()) { 1396176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // The higher-order double comes first, and so we need to truncate the 1397176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // pair to extract the overall sign. The order of the pair is the same 1398176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // in both little- and big-Endian modes. 1399176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines ArgWidth >>= 1; 1400176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines ArgIntTy = llvm::IntegerType::get(C, ArgWidth); 1401176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines BCArg = Builder.CreateTrunc(BCArg, ArgIntTy); 1402176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 1403ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman Value *ZeroCmp = llvm::Constant::getNullValue(ArgIntTy); 1404ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman Value *Result = Builder.CreateICmpSLT(BCArg, ZeroCmp); 1405ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman return RValue::get(Builder.CreateZExt(Result, ConvertType(E->getType()))); 1406ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman } 140777f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge case Builtin::BI__builtin_annotation: { 140877f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge llvm::Value *AnnVal = EmitScalarExpr(E->getArg(0)); 140977f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::annotation, 141077f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge AnnVal->getType()); 141177f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge 141277f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge // Get the annotation string, go through casts. Sema requires this to be a 141377f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge // non-wide string literal, potentially casted, so the cast<> is safe. 141477f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge const Expr *AnnotationStrExpr = E->getArg(1)->IgnoreParenCasts(); 1415cfa88f893915ceb8ae4ce2f17c46c24a4d67502fDmitri Gribenko StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString(); 141677f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge return RValue::get(EmitAnnotationCall(F, AnnVal, Str, E->getExprLoc())); 141777f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge } 1418ee76e7215d27c25bb053b71ae0a59119c08ccef9Michael Gottesman case Builtin::BI__builtin_addcb: 14190cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman case Builtin::BI__builtin_addcs: 14200cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman case Builtin::BI__builtin_addc: 14210cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman case Builtin::BI__builtin_addcl: 14227c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_addcll: 1423ee76e7215d27c25bb053b71ae0a59119c08ccef9Michael Gottesman case Builtin::BI__builtin_subcb: 14247c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_subcs: 14257c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_subc: 14267c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_subcl: 14277c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_subcll: { 14280cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman 14290cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // We translate all of these builtins from expressions of the form: 14300cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // int x = ..., y = ..., carryin = ..., carryout, result; 14310cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // result = __builtin_addc(x, y, carryin, &carryout); 14320cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // 14330cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // to LLVM IR of the form: 14340cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // 14350cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // %tmp1 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %x, i32 %y) 14360cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // %tmpsum1 = extractvalue {i32, i1} %tmp1, 0 14370cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // %carry1 = extractvalue {i32, i1} %tmp1, 1 14380cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // %tmp2 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %tmpsum1, 14390cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // i32 %carryin) 14400cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // %result = extractvalue {i32, i1} %tmp2, 0 14410cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // %carry2 = extractvalue {i32, i1} %tmp2, 1 14420cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // %tmp3 = or i1 %carry1, %carry2 14430cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // %tmp4 = zext i1 %tmp3 to i32 14440cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // store i32 %tmp4, i32* %carryout 14450cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman 14460cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // Scalarize our inputs. 14470cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *X = EmitScalarExpr(E->getArg(0)); 14480cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *Y = EmitScalarExpr(E->getArg(1)); 14490cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *Carryin = EmitScalarExpr(E->getArg(2)); 14500cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman std::pair<llvm::Value*, unsigned> CarryOutPtr = 14510cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman EmitPointerWithAlignment(E->getArg(3)); 14520cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman 14537c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman // Decide if we are lowering to a uadd.with.overflow or usub.with.overflow. 14547c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman llvm::Intrinsic::ID IntrinsicId; 14557c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman switch (BuiltinID) { 14567c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman default: llvm_unreachable("Unknown multiprecision builtin id."); 1457ee76e7215d27c25bb053b71ae0a59119c08ccef9Michael Gottesman case Builtin::BI__builtin_addcb: 14587c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_addcs: 14597c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_addc: 14607c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_addcl: 14617c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_addcll: 14627c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman IntrinsicId = llvm::Intrinsic::uadd_with_overflow; 14637c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman break; 1464ee76e7215d27c25bb053b71ae0a59119c08ccef9Michael Gottesman case Builtin::BI__builtin_subcb: 14657c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_subcs: 14667c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_subc: 14677c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_subcl: 14687c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_subcll: 14697c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman IntrinsicId = llvm::Intrinsic::usub_with_overflow; 14707c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman break; 14717c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman } 14720cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman 14730cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // Construct our resulting LLVM IR expression. 14740cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *Carry1; 14750cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *Sum1 = EmitOverflowIntrinsic(*this, IntrinsicId, 14760cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman X, Y, Carry1); 14770cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *Carry2; 14780cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *Sum2 = EmitOverflowIntrinsic(*this, IntrinsicId, 14790cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman Sum1, Carryin, Carry2); 14800cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *CarryOut = Builder.CreateZExt(Builder.CreateOr(Carry1, Carry2), 14810cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman X->getType()); 14820cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::StoreInst *CarryOutStore = Builder.CreateStore(CarryOut, 14830cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman CarryOutPtr.first); 14840cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman CarryOutStore->setAlignment(CarryOutPtr.second); 14850cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman return RValue::get(Sum2); 14860cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman } 148798d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_uadd_overflow: 148898d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_uaddl_overflow: 148998d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_uaddll_overflow: 149098d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_usub_overflow: 149198d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_usubl_overflow: 149298d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_usubll_overflow: 149398d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_umul_overflow: 149498d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_umull_overflow: 149598d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_umulll_overflow: 149698d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_sadd_overflow: 149798d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_saddl_overflow: 149898d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_saddll_overflow: 149998d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_ssub_overflow: 150098d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_ssubl_overflow: 150198d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_ssubll_overflow: 150298d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_smul_overflow: 150398d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_smull_overflow: 150498d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_smulll_overflow: { 150598d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman 150698d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman // We translate all of these builtins directly to the relevant llvm IR node. 150798d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman 150898d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman // Scalarize our inputs. 150998d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman llvm::Value *X = EmitScalarExpr(E->getArg(0)); 151098d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman llvm::Value *Y = EmitScalarExpr(E->getArg(1)); 151198d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman std::pair<llvm::Value *, unsigned> SumOutPtr = 151298d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman EmitPointerWithAlignment(E->getArg(2)); 151398d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman 151498d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman // Decide which of the overflow intrinsics we are lowering to: 151598d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman llvm::Intrinsic::ID IntrinsicId; 151698d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman switch (BuiltinID) { 151798d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman default: llvm_unreachable("Unknown security overflow builtin id."); 151898d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_uadd_overflow: 151998d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_uaddl_overflow: 152098d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_uaddll_overflow: 152198d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman IntrinsicId = llvm::Intrinsic::uadd_with_overflow; 152298d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman break; 152398d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_usub_overflow: 152498d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_usubl_overflow: 152598d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_usubll_overflow: 152698d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman IntrinsicId = llvm::Intrinsic::usub_with_overflow; 152798d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman break; 152898d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_umul_overflow: 152998d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_umull_overflow: 153098d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_umulll_overflow: 153198d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman IntrinsicId = llvm::Intrinsic::umul_with_overflow; 153298d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman break; 153398d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_sadd_overflow: 153498d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_saddl_overflow: 153598d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_saddll_overflow: 153698d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman IntrinsicId = llvm::Intrinsic::sadd_with_overflow; 153798d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman break; 153898d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_ssub_overflow: 153998d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_ssubl_overflow: 154098d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_ssubll_overflow: 154198d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman IntrinsicId = llvm::Intrinsic::ssub_with_overflow; 154298d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman break; 154398d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_smul_overflow: 154498d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_smull_overflow: 154598d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_smulll_overflow: 154698d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman IntrinsicId = llvm::Intrinsic::smul_with_overflow; 154798d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman break; 154898d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman } 154998d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman 155098d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman 155198d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman llvm::Value *Carry; 155298d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman llvm::Value *Sum = EmitOverflowIntrinsic(*this, IntrinsicId, X, Y, Carry); 155398d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman llvm::StoreInst *SumOutStore = Builder.CreateStore(Sum, SumOutPtr.first); 155498d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman SumOutStore->setAlignment(SumOutPtr.second); 155598d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman 155698d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman return RValue::get(Carry); 155798d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman } 15585154dce6388e3aaa445467030df7a45ed1211abeRichard Smith case Builtin::BI__builtin_addressof: 15595154dce6388e3aaa445467030df7a45ed1211abeRichard Smith return RValue::get(EmitLValue(E->getArg(0)).getAddress()); 1560c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case Builtin::BI__builtin_operator_new: 1561c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines return EmitBuiltinNewDeleteCall(FD->getType()->castAs<FunctionProtoType>(), 1562c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines E->getArg(0), false); 1563c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case Builtin::BI__builtin_operator_delete: 1564c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines return EmitBuiltinNewDeleteCall(FD->getType()->castAs<FunctionProtoType>(), 1565c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines E->getArg(0), true); 156627844533aad5c165523a5926424097699610d3f0Nico Weber case Builtin::BI__noop: 1567176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // __noop always evaluates to an integer literal zero. 1568176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return RValue::get(ConstantInt::get(IntTy, 0)); 1569c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case Builtin::BI_InterlockedExchange: 1570c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case Builtin::BI_InterlockedExchangePointer: 1571c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E); 1572c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case Builtin::BI_InterlockedCompareExchangePointer: { 1573c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Type *RTy; 1574c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::IntegerType *IntType = 1575c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines IntegerType::get(getLLVMContext(), 1576c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines getContext().getTypeSize(E->getType())); 1577c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Type *IntPtrType = IntType->getPointerTo(); 1578c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 1579c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Destination = 1580c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), IntPtrType); 1581c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 1582c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Exchange = EmitScalarExpr(E->getArg(1)); 1583c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines RTy = Exchange->getType(); 1584c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Exchange = Builder.CreatePtrToInt(Exchange, IntType); 1585c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 1586c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Comparand = 1587c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Builder.CreatePtrToInt(EmitScalarExpr(E->getArg(2)), IntType); 1588c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 1589c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines auto Result = Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange, 1590c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines SequentiallyConsistent, 1591c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines SequentiallyConsistent); 1592c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Result->setVolatile(true); 1593c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 1594c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines return RValue::get(Builder.CreateIntToPtr(Builder.CreateExtractValue(Result, 1595c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 0), 1596c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines RTy)); 1597c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines } 1598651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case Builtin::BI_InterlockedCompareExchange: { 1599651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AtomicCmpXchgInst *CXI = Builder.CreateAtomicCmpXchg( 1600651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines EmitScalarExpr(E->getArg(0)), 1601651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines EmitScalarExpr(E->getArg(2)), 1602651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines EmitScalarExpr(E->getArg(1)), 1603651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SequentiallyConsistent, 1604651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SequentiallyConsistent); 1605651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CXI->setVolatile(true); 1606c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines return RValue::get(Builder.CreateExtractValue(CXI, 0)); 1607651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 1608651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case Builtin::BI_InterlockedIncrement: { 1609651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( 1610651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AtomicRMWInst::Add, 1611651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines EmitScalarExpr(E->getArg(0)), 1612651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ConstantInt::get(Int32Ty, 1), 1613651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::SequentiallyConsistent); 1614651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines RMWI->setVolatile(true); 1615651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return RValue::get(Builder.CreateAdd(RMWI, ConstantInt::get(Int32Ty, 1))); 1616651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 1617651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case Builtin::BI_InterlockedDecrement: { 1618651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( 1619651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AtomicRMWInst::Sub, 1620651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines EmitScalarExpr(E->getArg(0)), 1621651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ConstantInt::get(Int32Ty, 1), 1622651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::SequentiallyConsistent); 1623651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines RMWI->setVolatile(true); 1624651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return RValue::get(Builder.CreateSub(RMWI, ConstantInt::get(Int32Ty, 1))); 1625651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 1626651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case Builtin::BI_InterlockedExchangeAdd: { 1627651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( 1628651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AtomicRMWInst::Add, 1629651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines EmitScalarExpr(E->getArg(0)), 1630651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines EmitScalarExpr(E->getArg(1)), 1631651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::SequentiallyConsistent); 1632651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines RMWI->setVolatile(true); 1633651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return RValue::get(RMWI); 1634651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 1635176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__readfsdword: { 1636176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *IntToPtr = 1637176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Builder.CreateIntToPtr(EmitScalarExpr(E->getArg(0)), 1638176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::PointerType::get(CGM.Int32Ty, 257)); 1639176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines LoadInst *Load = 1640176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Builder.CreateAlignedLoad(IntToPtr, /*Align=*/4, /*isVolatile=*/true); 1641176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return RValue::get(Load); 1642176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 16437ea2e3f6aae9b7511686d3d26dee690fee81c3aaNate Begeman } 16441eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1645a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // If this is an alias for a lib function (e.g. __builtin_sin), emit 1646a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // the call using the normal call path, but using the unmangled 1647a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // version of the function name. 1648a45680b7e7c49ea9893c6cff585984f3e4120366John McCall if (getContext().BuiltinInfo.isLibFunction(BuiltinID)) 1649a45680b7e7c49ea9893c6cff585984f3e4120366John McCall return emitLibraryCall(*this, FD, E, 1650a45680b7e7c49ea9893c6cff585984f3e4120366John McCall CGM.getBuiltinLibFunction(FD, BuiltinID)); 1651258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 1652a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // If this is a predefined lib function (e.g. malloc), emit the call 1653a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // using exactly the normal call path. 1654a45680b7e7c49ea9893c6cff585984f3e4120366John McCall if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID)) 1655a45680b7e7c49ea9893c6cff585984f3e4120366John McCall return emitLibraryCall(*this, FD, E, EmitScalarExpr(E->getCallee())); 16561eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1657b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner // See if we have a target specific intrinsic. 1658a6f80ef997f0363386749087b325607eaa5adcfcDale Johannesen const char *Name = getContext().BuiltinInfo.GetName(BuiltinID); 165955cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic; 166055cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar if (const char *Prefix = 1661c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch())) { 166255cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar IntrinsicID = Intrinsic::getIntrinsicForGCCBuiltin(Prefix, Name); 1663c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines // NOTE we dont need to perform a compatibility flag check here since the 1664c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines // intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the 1665c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines // MS builtins via ALL_MS_LANGUAGES and are filtered earlier. 1666c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines if (IntrinsicID == Intrinsic::not_intrinsic) 1667c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix, Name); 1668c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines } 16691eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1670b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner if (IntrinsicID != Intrinsic::not_intrinsic) { 1671b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner SmallVector<Value*, 16> Args; 16721eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 167346c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner // Find out if any arguments are required to be integer constant 167446c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner // expressions. 167546c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner unsigned ICEArguments = 0; 167646c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner ASTContext::GetBuiltinTypeError Error; 167746c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); 167846c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner assert(Error == ASTContext::GE_None && "Should not codegen an error"); 167946c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner 1680b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner Function *F = CGM.getIntrinsic(IntrinsicID); 16812acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::FunctionType *FTy = F->getFunctionType(); 16821eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1683b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) { 168446c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner Value *ArgValue; 168546c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner // If this is a normal argument, just emit it as a scalar. 168646c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner if ((ICEArguments & (1 << i)) == 0) { 168746c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner ArgValue = EmitScalarExpr(E->getArg(i)); 168846c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner } else { 1689258f930227c1a102c9c22eee88df65f748863425Jim Grosbach // If this is required to be a constant, constant fold it so that we 169046c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner // know that the generated intrinsic gets a ConstantInt. 169146c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner llvm::APSInt Result; 169246c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result,getContext()); 169346c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner assert(IsConst && "Constant arg isn't actually constant?"); 169446c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner (void)IsConst; 1695d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall ArgValue = llvm::ConstantInt::get(getLLVMContext(), Result); 169646c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner } 16971eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1698b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner // If the intrinsic arg type is different from the builtin arg type 1699b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner // we need to do a bit cast. 17002acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *PTy = FTy->getParamType(i); 1701b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner if (PTy != ArgValue->getType()) { 1702b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner assert(PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) && 1703b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner "Must be able to losslessly bit cast to param"); 1704b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner ArgValue = Builder.CreateBitCast(ArgValue, PTy); 1705b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner } 17061eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1707b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner Args.push_back(ArgValue); 1708b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner } 17091eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 17104c7d9f1507d0f102bd4133bba63348636facd469Jay Foad Value *V = Builder.CreateCall(F, Args); 1711b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner QualType BuiltinRetType = E->getType(); 17121eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 17138b418685e9e4f02f4eb2a76e1ec063e07552b68dChris Lattner llvm::Type *RetTy = VoidTy; 1714258f930227c1a102c9c22eee88df65f748863425Jim Grosbach if (!BuiltinRetType->isVoidType()) 17158b418685e9e4f02f4eb2a76e1ec063e07552b68dChris Lattner RetTy = ConvertType(BuiltinRetType); 17161eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1717b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner if (RetTy != V->getType()) { 1718b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner assert(V->getType()->canLosslesslyBitCastTo(RetTy) && 1719b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner "Must be able to losslessly bit cast result type"); 1720b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner V = Builder.CreateBitCast(V, RetTy); 1721b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner } 17221eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1723b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner return RValue::get(V); 1724b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner } 17251eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1726b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner // See if we have a target specific builtin that needs to be lowered. 1727f02e9ddf5efc75917af712b3c7f909581205f0a5Daniel Dunbar if (Value *V = EmitTargetBuiltinExpr(BuiltinID, E)) 1728b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner return RValue::get(V); 17291eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1730488e993a135ce700b982bf099c3d6b856301d642Daniel Dunbar ErrorUnsupported(E, "builtin function"); 17311eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1732b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner // Unknown builtin, for now just dump it out and return undef. 17339d232c884ea9872d6555df0fd7359699819bc1f1John McCall return GetUndefRValue(E->getType()); 17341eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump} 1735564f1de67d7ba43646b8740db86d6269e3dfbe0bAnders Carlsson 1736f02e9ddf5efc75917af712b3c7f909581205f0a5Daniel DunbarValue *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID, 1737f02e9ddf5efc75917af712b3c7f909581205f0a5Daniel Dunbar const CallExpr *E) { 173864aa4b3ec7e62288e2e66c1935487ece995ca94bJohn McCall switch (getTarget().getTriple().getArch()) { 17392752c0137d95aa2f4ee1cdff4b564bac842e041bChris Lattner case llvm::Triple::arm: 1740651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case llvm::Triple::armeb: 17412752c0137d95aa2f4ee1cdff4b564bac842e041bChris Lattner case llvm::Triple::thumb: 1742651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case llvm::Triple::thumbeb: 17432752c0137d95aa2f4ee1cdff4b564bac842e041bChris Lattner return EmitARMBuiltinExpr(BuiltinID, E); 17446bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case llvm::Triple::aarch64: 17456bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case llvm::Triple::aarch64_be: 17466bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitAArch64BuiltinExpr(BuiltinID, E); 174755cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar case llvm::Triple::x86: 174855cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar case llvm::Triple::x86_64: 1749f02e9ddf5efc75917af712b3c7f909581205f0a5Daniel Dunbar return EmitX86BuiltinExpr(BuiltinID, E); 175055cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar case llvm::Triple::ppc: 175155cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar case llvm::Triple::ppc64: 1752ea7fb0ce25acc04664a2e7c2b24af03cef2c0d1fBill Schmidt case llvm::Triple::ppc64le: 1753f02e9ddf5efc75917af712b3c7f909581205f0a5Daniel Dunbar return EmitPPCBuiltinExpr(BuiltinID, E); 1754c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case llvm::Triple::r600: 1755c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines return EmitR600BuiltinExpr(BuiltinID, E); 175655cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar default: 17576bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 175855cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar } 1759f02e9ddf5efc75917af712b3c7f909581205f0a5Daniel Dunbar} 1760f02e9ddf5efc75917af712b3c7f909581205f0a5Daniel Dunbar 17618b418685e9e4f02f4eb2a76e1ec063e07552b68dChris Lattnerstatic llvm::VectorType *GetNeonType(CodeGenFunction *CGF, 1762aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu NeonTypeFlags TypeFlags, 1763aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu bool V1Ty=false) { 176483084c863572b48579767a4dd5dc686e1a8d669dNAKAMURA Takumi int IsQuad = TypeFlags.isQuad(); 176583084c863572b48579767a4dd5dc686e1a8d669dNAKAMURA Takumi switch (TypeFlags.getEltType()) { 1766da95f73b59f9af964e33725c515139d34c90c863Bob Wilson case NeonTypeFlags::Int8: 1767da95f73b59f9af964e33725c515139d34c90c863Bob Wilson case NeonTypeFlags::Poly8: 1768aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu return llvm::VectorType::get(CGF->Int8Ty, V1Ty ? 1 : (8 << IsQuad)); 1769da95f73b59f9af964e33725c515139d34c90c863Bob Wilson case NeonTypeFlags::Int16: 1770da95f73b59f9af964e33725c515139d34c90c863Bob Wilson case NeonTypeFlags::Poly16: 1771da95f73b59f9af964e33725c515139d34c90c863Bob Wilson case NeonTypeFlags::Float16: 1772aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu return llvm::VectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad)); 1773da95f73b59f9af964e33725c515139d34c90c863Bob Wilson case NeonTypeFlags::Int32: 1774aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu return llvm::VectorType::get(CGF->Int32Ty, V1Ty ? 1 : (2 << IsQuad)); 1775da95f73b59f9af964e33725c515139d34c90c863Bob Wilson case NeonTypeFlags::Int64: 1776624bb5e59dbcc24efeee7dff12c9b48d2b5077e9Kevin Qin case NeonTypeFlags::Poly64: 1777aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu return llvm::VectorType::get(CGF->Int64Ty, V1Ty ? 1 : (1 << IsQuad)); 1778651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NeonTypeFlags::Poly128: 1779651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: i128 and f128 doesn't get fully support in Clang and llvm. 1780651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // There is a lot of i128 and f128 API missing. 1781651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // so we use v16i8 to represent poly128 and get pattern matched. 1782651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return llvm::VectorType::get(CGF->Int8Ty, 16); 1783da95f73b59f9af964e33725c515139d34c90c863Bob Wilson case NeonTypeFlags::Float32: 1784aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu return llvm::VectorType::get(CGF->FloatTy, V1Ty ? 1 : (2 << IsQuad)); 1785b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover case NeonTypeFlags::Float64: 1786aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu return llvm::VectorType::get(CGF->DoubleTy, V1Ty ? 1 : (1 << IsQuad)); 1787561d3abc881033776ece385a01a510e1cbc1fa92David Blaikie } 17880cd6bd62f3f7ee42f08ad130395ac65564768990Benjamin Kramer llvm_unreachable("Unknown vector element type!"); 1789998622c10198a25ba06c93d7e908f88ba0acc920Nate Begeman} 1790998622c10198a25ba06c93d7e908f88ba0acc920Nate Begeman 1791cf55652cf668c1402eee0b12edd2e5a1bc34d7a1Bob WilsonValue *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C) { 1792d075c01c359b9cc120c3accc7166990f9f4ac423Nate Begeman unsigned nElts = cast<llvm::VectorType>(V->getType())->getNumElements(); 17932ce8842641cc312628c4be836d34eb250e7c3f78Chris Lattner Value* SV = llvm::ConstantVector::getSplat(nElts, C); 1794d075c01c359b9cc120c3accc7166990f9f4ac423Nate Begeman return Builder.CreateShuffleVector(V, V, SV, "lane"); 1795d075c01c359b9cc120c3accc7166990f9f4ac423Nate Begeman} 1796d075c01c359b9cc120c3accc7166990f9f4ac423Nate Begeman 179730d91718a676177f0d0d0210ce4fdb4f616df6e5Nate BegemanValue *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops, 1798db3d4d036037f379f12643e067b229862d61e932Bob Wilson const char *name, 179961eecf5aea018ea65c9ab0bccacd2996b15c632dNate Begeman unsigned shift, bool rightshift) { 180030d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman unsigned j = 0; 180130d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end(); 180230d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman ai != ae; ++ai, ++j) 180361eecf5aea018ea65c9ab0bccacd2996b15c632dNate Begeman if (shift > 0 && shift == j) 180461eecf5aea018ea65c9ab0bccacd2996b15c632dNate Begeman Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift); 180561eecf5aea018ea65c9ab0bccacd2996b15c632dNate Begeman else 180661eecf5aea018ea65c9ab0bccacd2996b15c632dNate Begeman Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name); 180730d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman 18084c7d9f1507d0f102bd4133bba63348636facd469Jay Foad return Builder.CreateCall(F, Ops, name); 180930d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman} 181030d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman 1811258f930227c1a102c9c22eee88df65f748863425Jim GrosbachValue *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty, 1812464ccb68f22a7e1c0a2844551c16f721540c91c3Nate Begeman bool neg) { 18132ce8842641cc312628c4be836d34eb250e7c3f78Chris Lattner int SV = cast<ConstantInt>(V)->getSExtValue(); 1814258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 18152acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::VectorType *VTy = cast<llvm::VectorType>(Ty); 1816464ccb68f22a7e1c0a2844551c16f721540c91c3Nate Begeman llvm::Constant *C = ConstantInt::get(VTy->getElementType(), neg ? -SV : SV); 18172ce8842641cc312628c4be836d34eb250e7c3f78Chris Lattner return llvm::ConstantVector::getSplat(VTy->getNumElements(), C); 1818464ccb68f22a7e1c0a2844551c16f721540c91c3Nate Begeman} 1819464ccb68f22a7e1c0a2844551c16f721540c91c3Nate Begeman 18207f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville// \brief Right-shift a vector by a constant. 18217f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la VieuvilleValue *CodeGenFunction::EmitNeonRShiftImm(Value *Vec, Value *Shift, 18227f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville llvm::Type *Ty, bool usgn, 18237f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville const char *name) { 18247f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville llvm::VectorType *VTy = cast<llvm::VectorType>(Ty); 18257f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville 18267f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville int ShiftAmt = cast<ConstantInt>(Shift)->getSExtValue(); 18277f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville int EltSize = VTy->getScalarSizeInBits(); 18287f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville 18297f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville Vec = Builder.CreateBitCast(Vec, Ty); 18307f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville 18317f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville // lshr/ashr are undefined when the shift amount is equal to the vector 18327f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville // element size. 18337f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville if (ShiftAmt == EltSize) { 18347f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville if (usgn) { 18357f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville // Right-shifting an unsigned value by its size yields 0. 18367f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville llvm::Constant *Zero = ConstantInt::get(VTy->getElementType(), 0); 18377f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville return llvm::ConstantVector::getSplat(VTy->getNumElements(), Zero); 18387f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville } else { 18397f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville // Right-shifting a signed value by its size is equivalent 18407f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville // to a shift of size-1. 18417f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville --ShiftAmt; 18427f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt); 18437f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville } 18447f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville } 18457f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville 18467f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville Shift = EmitNeonShiftVector(Shift, Ty, false); 18477f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville if (usgn) 18487f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville return Builder.CreateLShr(Vec, Shift, name); 18497f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville else 18507f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville return Builder.CreateAShr(Vec, Shift, name); 18517f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville} 18527f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville 185306b6c589a5fff8e5476fe2b4cd6a660af71bfdddBob Wilson/// GetPointeeAlignment - Given an expression with a pointer type, find the 185406b6c589a5fff8e5476fe2b4cd6a660af71bfdddBob Wilson/// alignment of the type referenced by the pointer. Skip over implicit 185506b6c589a5fff8e5476fe2b4cd6a660af71bfdddBob Wilson/// casts. 1856ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedmanstd::pair<llvm::Value*, unsigned> 1857ea93e40785ffeadfac66b948c95f9490ec26207aEli FriedmanCodeGenFunction::EmitPointerWithAlignment(const Expr *Addr) { 1858ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman assert(Addr->getType()->isPointerType()); 1859ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman Addr = Addr->IgnoreParens(); 1860ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman if (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(Addr)) { 1861a484fc73ec6331bcaad092270b4ab9c8d1df23c3Eli Friedman if ((ICE->getCastKind() == CK_BitCast || ICE->getCastKind() == CK_NoOp) && 1862a484fc73ec6331bcaad092270b4ab9c8d1df23c3Eli Friedman ICE->getSubExpr()->getType()->isPointerType()) { 1863258f930227c1a102c9c22eee88df65f748863425Jim Grosbach std::pair<llvm::Value*, unsigned> Ptr = 1864ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman EmitPointerWithAlignment(ICE->getSubExpr()); 1865ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman Ptr.first = Builder.CreateBitCast(Ptr.first, 1866ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman ConvertType(Addr->getType())); 1867ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman return Ptr; 1868ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman } else if (ICE->getCastKind() == CK_ArrayToPointerDecay) { 1869ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman LValue LV = EmitLValue(ICE->getSubExpr()); 18708e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman unsigned Align = LV.getAlignment().getQuantity(); 18718e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman if (!Align) { 18728e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman // FIXME: Once LValues are fixed to always set alignment, 18738e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman // zap this code. 18748e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman QualType PtTy = ICE->getSubExpr()->getType(); 18758e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman if (!PtTy->isIncompleteType()) 18768e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman Align = getContext().getTypeAlignInChars(PtTy).getQuantity(); 18778e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman else 18788e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman Align = 1; 18798e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman } 18808e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman return std::make_pair(LV.getAddress(), Align); 1881d6e73569ccf09369f9bd2021d53c88e7bded06e3Chris Lattner } 1882ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman } 1883ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(Addr)) { 1884ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman if (UO->getOpcode() == UO_AddrOf) { 1885ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman LValue LV = EmitLValue(UO->getSubExpr()); 18868e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman unsigned Align = LV.getAlignment().getQuantity(); 18878e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman if (!Align) { 18888e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman // FIXME: Once LValues are fixed to always set alignment, 18898e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman // zap this code. 18908e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman QualType PtTy = UO->getSubExpr()->getType(); 18918e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman if (!PtTy->isIncompleteType()) 18928e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman Align = getContext().getTypeAlignInChars(PtTy).getQuantity(); 18938e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman else 18948e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman Align = 1; 18958e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman } 18968e4c189e6502cffb760b33f092e65effb1d10092Eli Friedman return std::make_pair(LV.getAddress(), Align); 189706b6c589a5fff8e5476fe2b4cd6a660af71bfdddBob Wilson } 189806b6c589a5fff8e5476fe2b4cd6a660af71bfdddBob Wilson } 1899f4c3db175101cbf94dad59419c3ed7aed51bf606Jay Foad 1900ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman unsigned Align = 1; 1901ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman QualType PtTy = Addr->getType()->getPointeeType(); 1902ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman if (!PtTy->isIncompleteType()) 1903ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman Align = getContext().getTypeAlignInChars(PtTy).getQuantity(); 1904ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman 1905ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman return std::make_pair(EmitScalarExpr(Addr), Align); 190606b6c589a5fff8e5476fe2b4cd6a660af71bfdddBob Wilson} 190706b6c589a5fff8e5476fe2b4cd6a660af71bfdddBob Wilson 1908651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesenum { 1909651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AddRetType = (1 << 0), 1910651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Add1ArgType = (1 << 1), 1911651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Add2ArgTypes = (1 << 2), 1912651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1913651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VectorizeRetType = (1 << 3), 1914651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VectorizeArgTypes = (1 << 4), 1915651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1916651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines InventFloatType = (1 << 5), 1917651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines UnsignedAlts = (1 << 6), 1918651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1919651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Use64BitVectors = (1 << 7), 1920651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Use128BitVectors = (1 << 8), 1921651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1922651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Vectorize1ArgType = Add1ArgType | VectorizeArgTypes, 1923651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VectorRet = AddRetType | VectorizeRetType, 1924651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VectorRetGetArgs01 = 1925651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AddRetType | Add2ArgTypes | VectorizeRetType | VectorizeArgTypes, 1926651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines FpCmpzModifiers = 1927651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AddRetType | VectorizeRetType | Add1ArgType | InventFloatType 1928651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines}; 1929651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1930651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines struct NeonIntrinsicInfo { 1931651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned BuiltinID; 1932651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned LLVMIntrinsic; 1933651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned AltLLVMIntrinsic; 1934651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const char *NameHint; 1935651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned TypeModifier; 1936651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1937651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool operator<(unsigned RHSBuiltinID) const { 1938651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return BuiltinID < RHSBuiltinID; 1939651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 1940651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines}; 1941651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1942651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#define NEONMAP0(NameBase) \ 1943651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines { NEON::BI__builtin_neon_ ## NameBase, 0, 0, #NameBase, 0 } 1944651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1945651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \ 1946651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines { NEON:: BI__builtin_neon_ ## NameBase, \ 1947651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Intrinsic::LLVMIntrinsic, 0, #NameBase, TypeModifier } 1948651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1949651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \ 1950651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines { NEON:: BI__builtin_neon_ ## NameBase, \ 1951651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \ 1952651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines #NameBase, TypeModifier } 1953651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1954651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic NeonIntrinsicInfo ARMSIMDIntrinsicMap [] = { 1955651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vabd_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts), 1956651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vabdq_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts), 1957651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vabs_v, arm_neon_vabs, 0), 1958651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vabsq_v, arm_neon_vabs, 0), 1959651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vaddhn_v), 1960651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vaesdq_v, arm_neon_aesd, 0), 1961651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vaeseq_v, arm_neon_aese, 0), 1962651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vaesimcq_v, arm_neon_aesimc, 0), 1963651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vaesmcq_v, arm_neon_aesmc, 0), 1964651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vbsl_v, arm_neon_vbsl, AddRetType), 1965651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vbslq_v, arm_neon_vbsl, AddRetType), 1966651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcage_v, arm_neon_vacge, 0), 1967651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcageq_v, arm_neon_vacge, 0), 1968651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcagt_v, arm_neon_vacgt, 0), 1969651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcagtq_v, arm_neon_vacgt, 0), 1970651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcale_v, arm_neon_vacge, 0), 1971651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcaleq_v, arm_neon_vacge, 0), 1972651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcalt_v, arm_neon_vacgt, 0), 1973651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcaltq_v, arm_neon_vacgt, 0), 1974651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcls_v, arm_neon_vcls, Add1ArgType), 1975651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vclsq_v, arm_neon_vcls, Add1ArgType), 1976651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vclz_v, ctlz, Add1ArgType), 1977651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vclzq_v, ctlz, Add1ArgType), 1978651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcnt_v, ctpop, Add1ArgType), 1979651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcntq_v, ctpop, Add1ArgType), 1980651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvt_f16_v, arm_neon_vcvtfp2hf, 0), 1981651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0), 1982651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvt_f32_v), 1983651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0), 1984651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0), 1985651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0), 1986651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0), 1987651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0), 1988651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvt_s32_v), 1989651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvt_s64_v), 1990651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvt_u32_v), 1991651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvt_u64_v), 1992651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0), 1993651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0), 1994651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0), 1995651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0), 1996651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0), 1997651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0), 1998651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0), 1999651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0), 2000651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0), 2001651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0), 2002651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0), 2003651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0), 2004651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0), 2005651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0), 2006651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0), 2007651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0), 2008651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0), 2009651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0), 2010651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0), 2011651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0), 2012651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0), 2013651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0), 2014651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0), 2015651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0), 2016651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0), 2017651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0), 2018651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0), 2019651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0), 2020651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0), 2021651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0), 2022651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0), 2023651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0), 2024651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvtq_f32_v), 2025651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0), 2026651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0), 2027651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0), 2028651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0), 2029651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0), 2030651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvtq_s32_v), 2031651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvtq_s64_v), 2032651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvtq_u32_v), 2033651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvtq_u64_v), 2034651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vext_v), 2035651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vextq_v), 2036651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vfma_v), 2037651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vfmaq_v), 2038651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vhadd_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts), 2039651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vhaddq_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts), 2040651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vhsub_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts), 2041651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vhsubq_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts), 2042651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vld1_dup_v), 2043651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld1_v, arm_neon_vld1, 0), 2044651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vld1q_dup_v), 2045651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld1q_v, arm_neon_vld1, 0), 2046651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0), 2047651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld2_v, arm_neon_vld2, 0), 2048651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0), 2049651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld2q_v, arm_neon_vld2, 0), 2050651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0), 2051651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld3_v, arm_neon_vld3, 0), 2052651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0), 2053651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld3q_v, arm_neon_vld3, 0), 2054651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0), 2055651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld4_v, arm_neon_vld4, 0), 2056651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0), 2057651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld4q_v, arm_neon_vld4, 0), 2058651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vmax_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts), 2059176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vmaxnm_v, arm_neon_vmaxnm, Add1ArgType), 2060176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vmaxnmq_v, arm_neon_vmaxnm, Add1ArgType), 2061651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vmaxq_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts), 2062651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vmin_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts), 2063176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vminnm_v, arm_neon_vminnm, Add1ArgType), 2064176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vminnmq_v, arm_neon_vminnm, Add1ArgType), 2065651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vminq_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts), 2066651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vmovl_v), 2067651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vmovn_v), 2068651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vmul_v, arm_neon_vmulp, Add1ArgType), 2069651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vmull_v), 2070651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vmulq_v, arm_neon_vmulp, Add1ArgType), 2071651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vpadal_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts), 2072651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vpadalq_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts), 2073651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vpadd_v, arm_neon_vpadd, Add1ArgType), 2074651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vpaddl_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts), 2075651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vpaddlq_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts), 2076651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vpaddq_v, arm_neon_vpadd, Add1ArgType), 2077651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vpmax_v, arm_neon_vpmaxu, arm_neon_vpmaxs, Add1ArgType | UnsignedAlts), 2078651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vpmin_v, arm_neon_vpminu, arm_neon_vpmins, Add1ArgType | UnsignedAlts), 2079651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqabs_v, arm_neon_vqabs, Add1ArgType), 2080651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqabsq_v, arm_neon_vqabs, Add1ArgType), 2081651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqadd_v, arm_neon_vqaddu, arm_neon_vqadds, Add1ArgType | UnsignedAlts), 2082651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqaddq_v, arm_neon_vqaddu, arm_neon_vqadds, Add1ArgType | UnsignedAlts), 2083651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqdmlal_v, arm_neon_vqdmull, arm_neon_vqadds, 0), 2084651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, arm_neon_vqsubs, 0), 2085651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqdmulh_v, arm_neon_vqdmulh, Add1ArgType), 2086651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqdmulhq_v, arm_neon_vqdmulh, Add1ArgType), 2087651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqdmull_v, arm_neon_vqdmull, Add1ArgType), 2088651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqmovn_v, arm_neon_vqmovnu, arm_neon_vqmovns, Add1ArgType | UnsignedAlts), 2089651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqmovun_v, arm_neon_vqmovnsu, Add1ArgType), 2090651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqneg_v, arm_neon_vqneg, Add1ArgType), 2091651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqnegq_v, arm_neon_vqneg, Add1ArgType), 2092651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqrdmulh_v, arm_neon_vqrdmulh, Add1ArgType), 2093651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqrdmulhq_v, arm_neon_vqrdmulh, Add1ArgType), 2094651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqrshl_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts), 2095651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqrshlq_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts), 2096651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqshl_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts), 2097651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqshl_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts), 2098651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqshlq_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts), 2099651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqshlq_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts), 2100176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0), 2101176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0), 2102651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqsub_v, arm_neon_vqsubu, arm_neon_vqsubs, Add1ArgType | UnsignedAlts), 2103651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqsubq_v, arm_neon_vqsubu, arm_neon_vqsubs, Add1ArgType | UnsignedAlts), 2104651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vraddhn_v, arm_neon_vraddhn, Add1ArgType), 2105651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0), 2106651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0), 2107651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vrecps_v, arm_neon_vrecps, Add1ArgType), 2108651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vrecpsq_v, arm_neon_vrecps, Add1ArgType), 2109651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vrhadd_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts), 2110651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vrhaddq_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts), 2111176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vrnd_v, arm_neon_vrintz, Add1ArgType), 2112176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vrnda_v, arm_neon_vrinta, Add1ArgType), 2113176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vrndaq_v, arm_neon_vrinta, Add1ArgType), 2114176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vrndm_v, arm_neon_vrintm, Add1ArgType), 2115176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vrndmq_v, arm_neon_vrintm, Add1ArgType), 2116176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vrndn_v, arm_neon_vrintn, Add1ArgType), 2117176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vrndnq_v, arm_neon_vrintn, Add1ArgType), 2118176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vrndp_v, arm_neon_vrintp, Add1ArgType), 2119176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vrndpq_v, arm_neon_vrintp, Add1ArgType), 2120176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vrndq_v, arm_neon_vrintz, Add1ArgType), 2121176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vrndx_v, arm_neon_vrintx, Add1ArgType), 2122176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vrndxq_v, arm_neon_vrintx, Add1ArgType), 2123651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vrshl_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts), 2124651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vrshlq_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts), 2125176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP2(vrshr_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts), 2126176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP2(vrshrq_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts), 2127651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0), 2128651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0), 2129651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vrsqrts_v, arm_neon_vrsqrts, Add1ArgType), 2130651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vrsqrtsq_v, arm_neon_vrsqrts, Add1ArgType), 2131651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vrsubhn_v, arm_neon_vrsubhn, Add1ArgType), 2132651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vsha1su0q_v, arm_neon_sha1su0, 0), 2133651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vsha1su1q_v, arm_neon_sha1su1, 0), 2134651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vsha256h2q_v, arm_neon_sha256h2, 0), 2135651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vsha256hq_v, arm_neon_sha256h, 0), 2136651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vsha256su0q_v, arm_neon_sha256su0, 0), 2137651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vsha256su1q_v, arm_neon_sha256su1, 0), 2138651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshl_n_v), 2139651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vshl_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts), 2140651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshll_n_v), 2141651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshlq_n_v), 2142651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vshlq_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts), 2143651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshr_n_v), 2144651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshrn_n_v), 2145651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshrq_n_v), 2146651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst1_v, arm_neon_vst1, 0), 2147651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst1q_v, arm_neon_vst1, 0), 2148651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0), 2149651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst2_v, arm_neon_vst2, 0), 2150651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0), 2151651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst2q_v, arm_neon_vst2, 0), 2152651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0), 2153651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst3_v, arm_neon_vst3, 0), 2154651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0), 2155651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst3q_v, arm_neon_vst3, 0), 2156651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0), 2157651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst4_v, arm_neon_vst4, 0), 2158651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0), 2159651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst4q_v, arm_neon_vst4, 0), 2160651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vsubhn_v), 2161651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vtrn_v), 2162651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vtrnq_v), 2163651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vtst_v), 2164651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vtstq_v), 2165651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vuzp_v), 2166651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vuzpq_v), 2167651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vzip_v), 2168651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vzipq_v) 2169651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines}; 2170651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 21716bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hinesstatic NeonIntrinsicInfo AArch64SIMDIntrinsicMap[] = { 21726bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vabs_v, aarch64_neon_abs, 0), 21736bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vabsq_v, aarch64_neon_abs, 0), 2174651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vaddhn_v), 21756bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaesdq_v, aarch64_crypto_aesd, 0), 21766bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaeseq_v, aarch64_crypto_aese, 0), 21776bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaesimcq_v, aarch64_crypto_aesimc, 0), 21786bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaesmcq_v, aarch64_crypto_aesmc, 0), 21796bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcage_v, aarch64_neon_facge, 0), 21806bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcageq_v, aarch64_neon_facge, 0), 21816bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcagt_v, aarch64_neon_facgt, 0), 21826bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0), 21836bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcale_v, aarch64_neon_facge, 0), 21846bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcaleq_v, aarch64_neon_facge, 0), 21856bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcalt_v, aarch64_neon_facgt, 0), 21866bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0), 21876bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcls_v, aarch64_neon_cls, Add1ArgType), 21886bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vclsq_v, aarch64_neon_cls, Add1ArgType), 2189651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vclz_v, ctlz, Add1ArgType), 2190651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vclzq_v, ctlz, Add1ArgType), 2191651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcnt_v, ctpop, Add1ArgType), 2192651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcntq_v, ctpop, Add1ArgType), 21936bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvt_f16_v, aarch64_neon_vcvtfp2hf, 0), 21946bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0), 2195651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvt_f32_v), 21966bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 21976bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 21986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0), 21996bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0), 22006bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0), 22016bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0), 2202651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvtq_f32_v), 22036bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 22046bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 22056bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0), 22066bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0), 22076bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0), 22086bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0), 22096bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtx_f32_v, aarch64_neon_fcvtxn, AddRetType | Add1ArgType), 2210651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vext_v), 2211651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vextq_v), 2212651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vfma_v), 2213651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vfmaq_v), 22146bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vhadd_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts), 22156bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vhaddq_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts), 22166bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vhsub_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts), 22176bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vhsubq_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts), 2218651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vmovl_v), 2219651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vmovn_v), 22206bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmul_v, aarch64_neon_pmul, Add1ArgType), 22216bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmulq_v, aarch64_neon_pmul, Add1ArgType), 22226bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpadd_v, aarch64_neon_addp, Add1ArgType), 22236bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vpaddl_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts), 22246bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vpaddlq_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts), 22256bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpaddq_v, aarch64_neon_addp, Add1ArgType), 22266bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqabs_v, aarch64_neon_sqabs, Add1ArgType), 22276bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqabsq_v, aarch64_neon_sqabs, Add1ArgType), 22286bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqadd_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts), 22296bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqaddq_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts), 22306bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0), 22316bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0), 22326bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqdmulh_v, aarch64_neon_sqdmulh, Add1ArgType), 22336bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqdmulhq_v, aarch64_neon_sqdmulh, Add1ArgType), 22346bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqdmull_v, aarch64_neon_sqdmull, Add1ArgType), 22356bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqmovn_v, aarch64_neon_uqxtn, aarch64_neon_sqxtn, Add1ArgType | UnsignedAlts), 22366bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovun_v, aarch64_neon_sqxtun, Add1ArgType), 22376bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqneg_v, aarch64_neon_sqneg, Add1ArgType), 22386bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqnegq_v, aarch64_neon_sqneg, Add1ArgType), 22396bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrdmulh_v, aarch64_neon_sqrdmulh, Add1ArgType), 22406bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrdmulhq_v, aarch64_neon_sqrdmulh, Add1ArgType), 22416bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqrshl_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts), 22426bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqrshlq_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts), 22436bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqshl_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl, UnsignedAlts), 22446bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqshl_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts), 22456bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqshlq_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl,UnsignedAlts), 22466bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqshlq_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts), 2247176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0), 2248176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0), 22496bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqsub_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts), 22506bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqsubq_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts), 22516bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vraddhn_v, aarch64_neon_raddhn, Add1ArgType), 22526bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0), 22536bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0), 22546bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrecps_v, aarch64_neon_frecps, Add1ArgType), 22556bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrecpsq_v, aarch64_neon_frecps, Add1ArgType), 22566bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vrhadd_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts), 22576bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vrhaddq_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts), 22586bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vrshl_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts), 22596bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vrshlq_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts), 2260176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP2(vrshr_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts), 2261176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP2(vrshrq_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts), 22626bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0), 22636bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0), 22646bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrsqrts_v, aarch64_neon_frsqrts, Add1ArgType), 22656bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrsqrtsq_v, aarch64_neon_frsqrts, Add1ArgType), 22666bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrsubhn_v, aarch64_neon_rsubhn, Add1ArgType), 22676bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha1su0q_v, aarch64_crypto_sha1su0, 0), 22686bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha1su1q_v, aarch64_crypto_sha1su1, 0), 22696bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha256h2q_v, aarch64_crypto_sha256h2, 0), 22706bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha256hq_v, aarch64_crypto_sha256h, 0), 22716bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha256su0q_v, aarch64_crypto_sha256su0, 0), 22726bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha256su1q_v, aarch64_crypto_sha256su1, 0), 2273651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshl_n_v), 22746bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vshl_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts), 2275651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshll_n_v), 2276651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshlq_n_v), 22776bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vshlq_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts), 2278651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshr_n_v), 2279651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshrn_n_v), 2280651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshrq_n_v), 2281651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vsubhn_v), 2282651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vtst_v), 2283651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vtstq_v), 2284651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines}; 2285651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 22866bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hinesstatic NeonIntrinsicInfo AArch64SISDIntrinsicMap[] = { 22876bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vabdd_f64, aarch64_sisd_fabd, Add1ArgType), 22886bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vabds_f32, aarch64_sisd_fabd, Add1ArgType), 22896bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vabsd_s64, aarch64_neon_abs, Add1ArgType), 22906bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddlv_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType), 22916bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddlv_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType), 22926bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddlvq_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType), 22936bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddlvq_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType), 22946bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddv_f32, aarch64_neon_faddv, AddRetType | Add1ArgType), 22956bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddv_s32, aarch64_neon_saddv, AddRetType | Add1ArgType), 22966bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddv_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType), 22976bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddvq_f32, aarch64_neon_faddv, AddRetType | Add1ArgType), 22986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddvq_f64, aarch64_neon_faddv, AddRetType | Add1ArgType), 22996bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddvq_s32, aarch64_neon_saddv, AddRetType | Add1ArgType), 23006bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddvq_s64, aarch64_neon_saddv, AddRetType | Add1ArgType), 23016bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddvq_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType), 23026bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddvq_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType), 23036bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcaged_f64, aarch64_neon_facge, AddRetType | Add1ArgType), 23046bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcages_f32, aarch64_neon_facge, AddRetType | Add1ArgType), 23056bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcagtd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType), 23066bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcagts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType), 23076bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcaled_f64, aarch64_neon_facge, AddRetType | Add1ArgType), 23086bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcales_f32, aarch64_neon_facge, AddRetType | Add1ArgType), 23096bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcaltd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType), 23106bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcalts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType), 23116bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtad_s64_f64, aarch64_neon_fcvtas, AddRetType | Add1ArgType), 23126bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtad_u64_f64, aarch64_neon_fcvtau, AddRetType | Add1ArgType), 23136bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtas_s32_f32, aarch64_neon_fcvtas, AddRetType | Add1ArgType), 23146bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtas_u32_f32, aarch64_neon_fcvtau, AddRetType | Add1ArgType), 23156bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtd_n_f64_s64, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType), 23166bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtd_n_f64_u64, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType), 23176bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtd_n_s64_f64, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType), 23186bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtd_n_u64_f64, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType), 23196bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtmd_s64_f64, aarch64_neon_fcvtms, AddRetType | Add1ArgType), 23206bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtmd_u64_f64, aarch64_neon_fcvtmu, AddRetType | Add1ArgType), 23216bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtms_s32_f32, aarch64_neon_fcvtms, AddRetType | Add1ArgType), 23226bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtms_u32_f32, aarch64_neon_fcvtmu, AddRetType | Add1ArgType), 23236bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtnd_s64_f64, aarch64_neon_fcvtns, AddRetType | Add1ArgType), 23246bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtnd_u64_f64, aarch64_neon_fcvtnu, AddRetType | Add1ArgType), 23256bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtns_s32_f32, aarch64_neon_fcvtns, AddRetType | Add1ArgType), 23266bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtns_u32_f32, aarch64_neon_fcvtnu, AddRetType | Add1ArgType), 23276bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtpd_s64_f64, aarch64_neon_fcvtps, AddRetType | Add1ArgType), 23286bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtpd_u64_f64, aarch64_neon_fcvtpu, AddRetType | Add1ArgType), 23296bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtps_s32_f32, aarch64_neon_fcvtps, AddRetType | Add1ArgType), 23306bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtps_u32_f32, aarch64_neon_fcvtpu, AddRetType | Add1ArgType), 23316bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvts_n_f32_s32, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType), 23326bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvts_n_f32_u32, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType), 23336bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvts_n_s32_f32, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType), 23346bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvts_n_u32_f32, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType), 23356bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0), 23366bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxnmv_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 23376bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxnmvq_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 23386bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxnmvq_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 23396bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxv_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 23406bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxv_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType), 23416bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxv_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType), 23426bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxvq_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 23436bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxvq_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 23446bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxvq_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType), 23456bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxvq_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType), 23466bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminnmv_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 23476bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminnmvq_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 23486bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminnmvq_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 23496bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminv_f32, aarch64_neon_fminv, AddRetType | Add1ArgType), 23506bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminv_s32, aarch64_neon_sminv, AddRetType | Add1ArgType), 23516bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminv_u32, aarch64_neon_uminv, AddRetType | Add1ArgType), 23526bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminvq_f32, aarch64_neon_fminv, AddRetType | Add1ArgType), 23536bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminvq_f64, aarch64_neon_fminv, AddRetType | Add1ArgType), 23546bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminvq_s32, aarch64_neon_sminv, AddRetType | Add1ArgType), 23556bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminvq_u32, aarch64_neon_uminv, AddRetType | Add1ArgType), 23566bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0), 23576bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmulxd_f64, aarch64_neon_fmulx, Add1ArgType), 23586bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmulxs_f32, aarch64_neon_fmulx, Add1ArgType), 23596bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpaddd_s64, aarch64_neon_uaddv, AddRetType | Add1ArgType), 23606bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpaddd_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType), 23616bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpmaxnmqd_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 23626bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpmaxnms_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 23636bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpmaxqd_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 23646bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpmaxs_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 23656bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpminnmqd_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 23666bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpminnms_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 23676bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpminqd_f64, aarch64_neon_fminv, AddRetType | Add1ArgType), 23686bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpmins_f32, aarch64_neon_fminv, AddRetType | Add1ArgType), 23696bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqabsb_s8, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors), 23706bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqabsd_s64, aarch64_neon_sqabs, Add1ArgType), 23716bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqabsh_s16, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors), 23726bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqabss_s32, aarch64_neon_sqabs, Add1ArgType), 23736bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqaddb_s8, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors), 23746bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqaddb_u8, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors), 23756bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqaddd_s64, aarch64_neon_sqadd, Add1ArgType), 23766bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqaddd_u64, aarch64_neon_uqadd, Add1ArgType), 23776bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqaddh_s16, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors), 23786bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqaddh_u16, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors), 23796bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqadds_s32, aarch64_neon_sqadd, Add1ArgType), 23806bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqadds_u32, aarch64_neon_uqadd, Add1ArgType), 23816bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqdmulhh_s16, aarch64_neon_sqdmulh, Vectorize1ArgType | Use64BitVectors), 23826bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqdmulhs_s32, aarch64_neon_sqdmulh, Add1ArgType), 23836bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqdmullh_s16, aarch64_neon_sqdmull, VectorRet | Use128BitVectors), 23846bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0), 23856bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovnd_s64, aarch64_neon_scalar_sqxtn, AddRetType | Add1ArgType), 23866bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovnd_u64, aarch64_neon_scalar_uqxtn, AddRetType | Add1ArgType), 23876bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovnh_s16, aarch64_neon_sqxtn, VectorRet | Use64BitVectors), 23886bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovnh_u16, aarch64_neon_uqxtn, VectorRet | Use64BitVectors), 23896bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovns_s32, aarch64_neon_sqxtn, VectorRet | Use64BitVectors), 23906bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovns_u32, aarch64_neon_uqxtn, VectorRet | Use64BitVectors), 23916bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovund_s64, aarch64_neon_scalar_sqxtun, AddRetType | Add1ArgType), 23926bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovunh_s16, aarch64_neon_sqxtun, VectorRet | Use64BitVectors), 23936bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovuns_s32, aarch64_neon_sqxtun, VectorRet | Use64BitVectors), 23946bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqnegb_s8, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors), 23956bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqnegd_s64, aarch64_neon_sqneg, Add1ArgType), 23966bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqnegh_s16, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors), 23976bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqnegs_s32, aarch64_neon_sqneg, Add1ArgType), 23986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrdmulhh_s16, aarch64_neon_sqrdmulh, Vectorize1ArgType | Use64BitVectors), 23996bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrdmulhs_s32, aarch64_neon_sqrdmulh, Add1ArgType), 24006bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshlb_s8, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors), 24016bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshlb_u8, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors), 24026bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshld_s64, aarch64_neon_sqrshl, Add1ArgType), 24036bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshld_u64, aarch64_neon_uqrshl, Add1ArgType), 24046bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshlh_s16, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors), 24056bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshlh_u16, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors), 24066bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshls_s32, aarch64_neon_sqrshl, Add1ArgType), 24076bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshls_u32, aarch64_neon_uqrshl, Add1ArgType), 24086bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshrnd_n_s64, aarch64_neon_sqrshrn, AddRetType), 24096bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshrnd_n_u64, aarch64_neon_uqrshrn, AddRetType), 24106bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshrnh_n_s16, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors), 24116bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshrnh_n_u16, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors), 24126bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshrns_n_s32, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors), 24136bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshrns_n_u32, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors), 24146bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshrund_n_s64, aarch64_neon_sqrshrun, AddRetType), 24156bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshrunh_n_s16, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors), 24166bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshruns_n_s32, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors), 24176bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlb_n_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 24186bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlb_n_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 24196bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlb_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 24206bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlb_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 24216bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshld_s64, aarch64_neon_sqshl, Add1ArgType), 24226bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshld_u64, aarch64_neon_uqshl, Add1ArgType), 24236bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlh_n_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 24246bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlh_n_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 24256bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlh_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 24266bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlh_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 24276bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshls_n_s32, aarch64_neon_sqshl, Add1ArgType), 24286bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshls_n_u32, aarch64_neon_uqshl, Add1ArgType), 24296bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshls_s32, aarch64_neon_sqshl, Add1ArgType), 24306bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshls_u32, aarch64_neon_uqshl, Add1ArgType), 24316bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlub_n_s8, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors), 24326bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshluh_n_s16, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors), 24336bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlus_n_s32, aarch64_neon_sqshlu, Add1ArgType), 24346bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshrnd_n_s64, aarch64_neon_sqshrn, AddRetType), 24356bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshrnd_n_u64, aarch64_neon_uqshrn, AddRetType), 24366bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshrnh_n_s16, aarch64_neon_sqshrn, VectorRet | Use64BitVectors), 24376bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshrnh_n_u16, aarch64_neon_uqshrn, VectorRet | Use64BitVectors), 24386bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshrns_n_s32, aarch64_neon_sqshrn, VectorRet | Use64BitVectors), 24396bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshrns_n_u32, aarch64_neon_uqshrn, VectorRet | Use64BitVectors), 24406bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshrund_n_s64, aarch64_neon_sqshrun, AddRetType), 24416bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshrunh_n_s16, aarch64_neon_sqshrun, VectorRet | Use64BitVectors), 24426bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshruns_n_s32, aarch64_neon_sqshrun, VectorRet | Use64BitVectors), 24436bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqsubb_s8, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors), 24446bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqsubb_u8, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors), 24456bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqsubd_s64, aarch64_neon_sqsub, Add1ArgType), 24466bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqsubd_u64, aarch64_neon_uqsub, Add1ArgType), 24476bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqsubh_s16, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors), 24486bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqsubh_u16, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors), 24496bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqsubs_s32, aarch64_neon_sqsub, Add1ArgType), 24506bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqsubs_u32, aarch64_neon_uqsub, Add1ArgType), 24516bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrecped_f64, aarch64_neon_frecpe, Add1ArgType), 24526bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrecpes_f32, aarch64_neon_frecpe, Add1ArgType), 24536bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrecpxd_f64, aarch64_neon_frecpx, Add1ArgType), 24546bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrecpxs_f32, aarch64_neon_frecpx, Add1ArgType), 24556bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrshld_s64, aarch64_neon_srshl, Add1ArgType), 24566bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrshld_u64, aarch64_neon_urshl, Add1ArgType), 24576bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrsqrted_f64, aarch64_neon_frsqrte, Add1ArgType), 24586bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrsqrtes_f32, aarch64_neon_frsqrte, Add1ArgType), 24596bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrsqrtsd_f64, aarch64_neon_frsqrts, Add1ArgType), 24606bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrsqrtss_f32, aarch64_neon_frsqrts, Add1ArgType), 24616bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0), 24626bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0), 24636bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0), 24646bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0), 24656bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vshld_s64, aarch64_neon_sshl, Add1ArgType), 24666bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vshld_u64, aarch64_neon_ushl, Add1ArgType), 24676bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vslid_n_s64, aarch64_neon_vsli, Vectorize1ArgType), 24686bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vslid_n_u64, aarch64_neon_vsli, Vectorize1ArgType), 24696bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsqaddb_u8, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors), 24706bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsqaddd_u64, aarch64_neon_usqadd, Add1ArgType), 24716bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsqaddh_u16, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors), 24726bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsqadds_u32, aarch64_neon_usqadd, Add1ArgType), 24736bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsrid_n_s64, aarch64_neon_vsri, Vectorize1ArgType), 24746bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsrid_n_u64, aarch64_neon_vsri, Vectorize1ArgType), 24756bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vuqaddb_s8, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors), 24766bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vuqaddd_s64, aarch64_neon_suqadd, Add1ArgType), 24776bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vuqaddh_s16, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors), 24786bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vuqadds_s32, aarch64_neon_suqadd, Add1ArgType), 2479651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines}; 2480651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2481651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#undef NEONMAP0 2482651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#undef NEONMAP1 2483651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#undef NEONMAP2 2484651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2485651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic bool NEONSIMDIntrinsicsProvenSorted = false; 2486651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 24876bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hinesstatic bool AArch64SIMDIntrinsicsProvenSorted = false; 24886bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hinesstatic bool AArch64SISDIntrinsicsProvenSorted = false; 2489651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2490651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2491651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic const NeonIntrinsicInfo * 2492c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen HinesfindNeonIntrinsicInMap(ArrayRef<NeonIntrinsicInfo> IntrinsicMap, 2493651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned BuiltinID, bool &MapProvenSorted) { 2494651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2495651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#ifndef NDEBUG 2496651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (!MapProvenSorted) { 2497651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: use std::is_sorted once C++11 is allowed 2498651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines for (unsigned i = 0; i < IntrinsicMap.size() - 1; ++i) 2499651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines assert(IntrinsicMap[i].BuiltinID <= IntrinsicMap[i + 1].BuiltinID); 2500651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines MapProvenSorted = true; 2501651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2502651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#endif 2503651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2504651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const NeonIntrinsicInfo *Builtin = 2505651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines std::lower_bound(IntrinsicMap.begin(), IntrinsicMap.end(), BuiltinID); 2506651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2507651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID) 2508651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builtin; 2509651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 25106bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 2511651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 2512651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2513651f13cea278ec967336033dd032faef0e9fc2ecStephen HinesFunction *CodeGenFunction::LookupNeonLLVMIntrinsic(unsigned IntrinsicID, 2514651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Modifier, 2515651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *ArgType, 2516651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const CallExpr *E) { 2517651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines int VectorSize = 0; 2518651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Modifier & Use64BitVectors) 2519651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VectorSize = 64; 2520651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines else if (Modifier & Use128BitVectors) 2521651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VectorSize = 128; 2522651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2523651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Return type. 2524651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<llvm::Type *, 3> Tys; 2525651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Modifier & AddRetType) { 2526651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty = ConvertType(E->getCallReturnType()); 2527651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Modifier & VectorizeRetType) 2528651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::VectorType::get( 2529651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1); 2530651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2531651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Tys.push_back(Ty); 2532651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2533651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2534651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Arguments. 2535651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Modifier & VectorizeArgTypes) { 2536651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1; 2537651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ArgType = llvm::VectorType::get(ArgType, Elts); 2538651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2539651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2540651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Modifier & (Add1ArgType | Add2ArgTypes)) 2541651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Tys.push_back(ArgType); 2542651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2543651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Modifier & Add2ArgTypes) 2544651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Tys.push_back(ArgType); 2545651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2546651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Modifier & InventFloatType) 2547651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Tys.push_back(FloatTy); 2548651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2549651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return CGM.getIntrinsic(IntrinsicID, Tys); 2550651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 2551651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2552651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic Value *EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, 2553651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const NeonIntrinsicInfo &SISDInfo, 2554651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVectorImpl<Value *> &Ops, 2555651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const CallExpr *E) { 2556651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned BuiltinID = SISDInfo.BuiltinID; 2557651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned int Int = SISDInfo.LLVMIntrinsic; 2558651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Modifier = SISDInfo.TypeModifier; 2559651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const char *s = SISDInfo.NameHint; 2560651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2561651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 2562651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_s64: 2563651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_u64: 2564651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcles_f32: 2565651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_f64: 2566651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_s64: 2567651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_u64: 2568651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclts_f32: 2569651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_f64: 2570651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcales_f32: 2571651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcaled_f64: 2572651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcalts_f32: 2573651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcaltd_f64: 2574651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Only one direction of comparisons actually exist, cmle is actually a cmge 2575651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // with swapped operands. The table gives us the right intrinsic but we 2576651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // still need to do the swap. 2577651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines std::swap(Ops[0], Ops[1]); 2578651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 2579651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2580651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2581651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines assert(Int && "Generic code assumes a valid intrinsic"); 2582651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2583651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Determine the type(s) of this overloaded AArch64 intrinsic. 2584651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const Expr *Arg = E->getArg(0); 2585651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *ArgTy = CGF.ConvertType(Arg->getType()); 2586651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGF.LookupNeonLLVMIntrinsic(Int, Modifier, ArgTy, E); 2587651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2588651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines int j = 0; 2589c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines ConstantInt *C0 = ConstantInt::get(CGF.SizeTy, 0); 2590651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end(); 2591651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ai != ae; ++ai, ++j) { 2592651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *ArgTy = ai->getType(); 2593651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Ops[j]->getType()->getPrimitiveSizeInBits() == 2594651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ArgTy->getPrimitiveSizeInBits()) 2595651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines continue; 2596651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2597651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy()); 2598651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The constant argument to an _n_ intrinsic always has Int32Ty, so truncate 2599651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // it before inserting. 2600651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[j] = 2601651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.CreateTruncOrBitCast(Ops[j], ArgTy->getVectorElementType()); 2602651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[j] = 2603651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.CreateInsertElement(UndefValue::get(ArgTy), Ops[j], C0); 2604651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2605651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2606651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Result = CGF.EmitNeonCall(F, Ops, s); 2607651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *ResultType = CGF.ConvertType(E->getType()); 2608651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (ResultType->getPrimitiveSizeInBits() < 2609651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Result->getType()->getPrimitiveSizeInBits()) 2610651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return CGF.Builder.CreateExtractElement(Result, C0); 2611651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2612651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return CGF.Builder.CreateBitCast(Result, ResultType, s); 2613651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 2614651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2615651f13cea278ec967336033dd032faef0e9fc2ecStephen HinesValue *CodeGenFunction::EmitCommonNeonBuiltinExpr( 2616651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic, 2617651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const char *NameHint, unsigned Modifier, const CallExpr *E, 2618651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVectorImpl<llvm::Value *> &Ops, llvm::Value *Align) { 26192e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu // Get the last argument, which specifies the vector type. 2620651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::APSInt NeonTypeConst; 26212e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu const Expr *Arg = E->getArg(E->getNumArgs() - 1); 2622651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (!Arg->isIntegerConstantExpr(NeonTypeConst, getContext())) 26236bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 26242e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu 26252e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu // Determine the type of this overloaded NEON intrinsic. 2626651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags Type(NeonTypeConst.getZExtValue()); 2627651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Usgn = Type.isUnsigned(); 2628651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Quad = Type.isQuad(); 2629651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2630651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType *VTy = GetNeonType(this, Type); 26312e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu llvm::Type *Ty = VTy; 26322e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu if (!Ty) 26336bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 26342e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu 2635651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Int = LLVMIntrinsic; 2636651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if ((Modifier & UnsignedAlts) && !Usgn) 2637651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = AltLLVMIntrinsic; 26382e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu 26392e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu switch (BuiltinID) { 2640651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines default: break; 2641651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vabs_v: 2642651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vabsq_v: 2643651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (VTy->getElementType()->isFloatingPointTy()) 2644651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs"); 2645651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vabs"); 2646651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddhn_v: { 2647651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType *SrcTy = 2648651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::getExtendedElementVectorType(VTy); 2649651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2650651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // %sum = add <4 x i32> %lhs, %rhs 2651651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 2652651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); 2653651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn"); 2654651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2655651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16> 2656651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Constant *ShiftAmt = ConstantInt::get(SrcTy->getElementType(), 2657651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SrcTy->getScalarSizeInBits() / 2); 2658651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ShiftAmt = ConstantVector::getSplat(VTy->getNumElements(), ShiftAmt); 2659651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn"); 2660651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2661651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // %res = trunc <4 x i32> %high to <4 x i16> 2662651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], VTy, "vaddhn"); 26632e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu } 2664651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcale_v: 2665651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcaleq_v: 2666651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcalt_v: 2667651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcaltq_v: 2668651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines std::swap(Ops[0], Ops[1]); 2669651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcage_v: 2670651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcageq_v: 2671651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcagt_v: 2672651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcagtq_v: { 2673651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *VecFlt = llvm::VectorType::get( 2674651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy->getScalarSizeInBits() == 32 ? FloatTy : DoubleTy, 2675651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy->getNumElements()); 2676651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[] = { VTy, VecFlt }; 2677651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); 2678651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(F, Ops, NameHint); 2679651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2680651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclz_v: 2681651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclzq_v: 2682651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // We generate target-independent intrinsic, which needs a second argument 2683651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // for whether or not clz of zero is undefined; on ARM it isn't. 2684651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef())); 2685651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 2686651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_f32_v: 2687651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_f32_v: 2688651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 2689651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, Quad)); 2690651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") 2691651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); 2692651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_n_f32_v: 2693651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_n_f64_v: 2694651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_n_f32_v: 2695651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_n_f64_v: { 2696651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Double = 2697651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 2698651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *FloatTy = 2699651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines GetNeonType(this, NeonTypeFlags(Double ? NeonTypeFlags::Float64 2700651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : NeonTypeFlags::Float32, 2701651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines false, Quad)); 2702651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { FloatTy, Ty }; 2703651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic; 2704651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(Int, Tys); 2705651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(F, Ops, "vcvt_n"); 27062e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu } 2707651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_n_s32_v: 2708651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_n_u32_v: 2709651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_n_s64_v: 2710651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_n_u64_v: 2711651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_n_s32_v: 2712651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_n_u32_v: 2713651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_n_s64_v: 2714651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_n_u64_v: { 2715651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Double = 2716651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 2717651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *FloatTy = 2718651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines GetNeonType(this, NeonTypeFlags(Double ? NeonTypeFlags::Float64 2719651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : NeonTypeFlags::Float32, 2720651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines false, Quad)); 2721651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, FloatTy }; 2722651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); 2723651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(F, Ops, "vcvt_n"); 27242e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu } 2725651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_s32_v: 2726651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_u32_v: 2727651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_s64_v: 2728651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_u64_v: 2729651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_s32_v: 2730651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_u32_v: 2731651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_s64_v: 2732651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_u64_v: { 2733651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Double = 2734651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 2735651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *FloatTy = 2736651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines GetNeonType(this, NeonTypeFlags(Double ? NeonTypeFlags::Float64 2737651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : NeonTypeFlags::Float32, 2738651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines false, Quad)); 2739651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy); 2740651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt") 2741651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : Builder.CreateFPToSI(Ops[0], Ty, "vcvt"); 27422e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu } 2743651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvta_s32_v: 2744651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvta_s64_v: 2745651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvta_u32_v: 2746651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvta_u64_v: 2747651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtaq_s32_v: 2748651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtaq_s64_v: 2749651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtaq_u32_v: 2750651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtaq_u64_v: 2751651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtn_s32_v: 2752651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtn_s64_v: 2753651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtn_u32_v: 2754651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtn_u64_v: 2755651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtnq_s32_v: 2756651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtnq_s64_v: 2757651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtnq_u32_v: 2758651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtnq_u64_v: 2759651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtp_s32_v: 2760651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtp_s64_v: 2761651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtp_u32_v: 2762651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtp_u64_v: 2763651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtpq_s32_v: 2764651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtpq_s64_v: 2765651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtpq_u32_v: 2766651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtpq_u64_v: 2767651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtm_s32_v: 2768651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtm_s64_v: 2769651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtm_u32_v: 2770651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtm_u64_v: 2771651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtmq_s32_v: 2772651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtmq_s64_v: 2773651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtmq_u32_v: 2774651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtmq_u64_v: { 2775651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Double = 2776651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 2777651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *InTy = 2778651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines GetNeonType(this, 2779651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags(Double ? NeonTypeFlags::Float64 2780651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : NeonTypeFlags::Float32, false, Quad)); 2781651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, InTy }; 2782651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint); 2783651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2784651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vext_v: 2785651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vextq_v: { 2786651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines int CV = cast<ConstantInt>(Ops[2])->getSExtValue(); 2787651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Constant*, 16> Indices; 2788651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) 2789651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Indices.push_back(ConstantInt::get(Int32Ty, i+CV)); 27902e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu 2791651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 2792651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 2793651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *SV = llvm::ConstantVector::get(Indices); 2794651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateShuffleVector(Ops[0], Ops[1], SV, "vext"); 27952e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu } 2796651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfma_v: 2797651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfmaq_v: { 2798651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 2799651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 2800651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 2801651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 28022e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu 2803651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // NEON intrinsic puts accumulator first, unlike the LLVM fma. 2804651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]); 2805651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2806651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_v: 2807651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_v: 2808651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Align); 2809651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vld1"); 2810651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_v: 2811651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2q_v: 2812651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_v: 2813651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3q_v: 2814651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_v: 2815651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4q_v: { 2816651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(LLVMIntrinsic, Ty); 2817651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall2(F, Ops[1], Align, NameHint); 2818651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 2819651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 2820651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateStore(Ops[1], Ops[0]); 2821651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2822651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_dup_v: 2823651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_dup_v: { 2824651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *V = UndefValue::get(Ty); 2825651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::PointerType::getUnqual(VTy->getElementType()); 2826651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 2827651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines LoadInst *Ld = Builder.CreateLoad(Ops[0]); 2828651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ld->setAlignment(cast<ConstantInt>(Align)->getZExtValue()); 2829c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Constant *CI = ConstantInt::get(SizeTy, 0); 2830651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateInsertElement(V, Ld, CI); 2831651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonSplat(Ops[0], CI); 2832651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2833651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_lane_v: 2834651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2q_lane_v: 2835651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_lane_v: 2836651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3q_lane_v: 2837651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_lane_v: 2838651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4q_lane_v: { 2839651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(LLVMIntrinsic, Ty); 2840651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines for (unsigned I = 2; I < Ops.size() - 1; ++I) 2841651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[I] = Builder.CreateBitCast(Ops[I], Ty); 2842651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Align); 2843651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), NameHint); 2844651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 2845651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 2846651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateStore(Ops[1], Ops[0]); 2847651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2848651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmovl_v: { 2849651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy); 2850651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], DTy); 2851651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Usgn) 2852651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateZExt(Ops[0], Ty, "vmovl"); 2853651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateSExt(Ops[0], Ty, "vmovl"); 2854651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2855651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmovn_v: { 2856651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *QTy = llvm::VectorType::getExtendedElementVectorType(VTy); 2857651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], QTy); 2858651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], Ty, "vmovn"); 2859651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2860651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmull_v: 2861651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: the integer vmull operations could be emitted in terms of pure 2862651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // LLVM IR (2 exts followed by a mul). Unfortunately LLVM has a habit of 2863651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // hoisting the exts outside loops. Until global ISel comes along that can 2864651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // see through such movement this leads to bad CodeGen. So we need an 2865651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // intrinsic for now. 2866651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls; 2867651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp : Int; 2868651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); 2869651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpadal_v: 2870651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpadalq_v: { 2871651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The source operand type has twice as many elements of half the size. 2872651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits(); 2873651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *EltTy = 2874651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), EltBits / 2); 2875651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *NarrowTy = 2876651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(EltTy, VTy->getNumElements() * 2); 2877651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, NarrowTy }; 2878651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint); 2879651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2880651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpaddl_v: 2881651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpaddlq_v: { 2882651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The source operand type has twice as many elements of half the size. 2883651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits(); 2884651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *EltTy = llvm::IntegerType::get(getLLVMContext(), EltBits / 2); 2885651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *NarrowTy = 2886651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(EltTy, VTy->getNumElements() * 2); 2887651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, NarrowTy }; 2888651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl"); 2889651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2890651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlal_v: 2891651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlsl_v: { 2892651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end()); 2893651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Mul = EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), 2894651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines MulOps, "vqdmlal"); 2895651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2896651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value *, 2> AccumOps; 2897651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AccumOps.push_back(Ops[0]); 2898651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AccumOps.push_back(Mul); 2899651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), 2900651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AccumOps, NameHint); 2901651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2902651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshl_n_v: 2903651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshlq_n_v: 2904651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n", 2905651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1, false); 2906176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case NEON::BI__builtin_neon_vqshlu_n_v: 2907176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case NEON::BI__builtin_neon_vqshluq_n_v: 2908176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshlu_n", 2909176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 1, false); 2910651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrecpe_v: 2911651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrecpeq_v: 2912651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrsqrte_v: 2913651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrsqrteq_v: 2914651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic; 2915651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint); 2916651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2917176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case NEON::BI__builtin_neon_vrshr_n_v: 2918176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case NEON::BI__builtin_neon_vrshrq_n_v: 2919176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n", 2920176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 1, true); 2921651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshl_n_v: 2922651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshlq_n_v: 2923651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false); 2924651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1], 2925651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vshl_n"); 2926651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshll_n_v: { 2927651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *SrcTy = llvm::VectorType::getTruncatedElementVectorType(VTy); 2928651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 2929651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Usgn) 2930651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateZExt(Ops[0], VTy); 2931651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines else 2932651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateSExt(Ops[0], VTy); 2933651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = EmitNeonShiftVector(Ops[1], VTy, false); 2934651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateShl(Ops[0], Ops[1], "vshll_n"); 2935651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2936651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshrn_n_v: { 2937651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *SrcTy = llvm::VectorType::getExtendedElementVectorType(VTy); 2938651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 2939651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy, false); 2940651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Usgn) 2941651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]); 2942651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines else 2943651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]); 2944651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n"); 2945651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2946651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshr_n_v: 2947651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshrq_n_v: 2948651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn, "vshr_n"); 2949651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_v: 2950651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_v: 2951651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2_v: 2952651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2q_v: 2953651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3_v: 2954651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3q_v: 2955651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4_v: 2956651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4q_v: 2957651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2_lane_v: 2958651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2q_lane_v: 2959651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3_lane_v: 2960651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3q_lane_v: 2961651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4_lane_v: 2962651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4q_lane_v: 2963651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Align); 2964651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, ""); 2965651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsubhn_v: { 2966651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType *SrcTy = 2967651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::getExtendedElementVectorType(VTy); 2968651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2969651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // %sum = add <4 x i32> %lhs, %rhs 2970651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 2971651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); 2972651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn"); 2973651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2974651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16> 2975651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Constant *ShiftAmt = ConstantInt::get(SrcTy->getElementType(), 2976651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SrcTy->getScalarSizeInBits() / 2); 2977651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ShiftAmt = ConstantVector::getSplat(VTy->getNumElements(), ShiftAmt); 2978651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn"); 2979651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2980651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // %res = trunc <4 x i32> %high to <4 x i16> 2981651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], VTy, "vsubhn"); 2982651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2983651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtrn_v: 2984651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtrnq_v: { 2985651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 2986651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 2987651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 29886bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *SV = nullptr; 2989651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2990651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines for (unsigned vi = 0; vi != 2; ++vi) { 2991651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Constant*, 16> Indices; 2992651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 2993651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Indices.push_back(Builder.getInt32(i+vi)); 2994651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Indices.push_back(Builder.getInt32(i+e+vi)); 2995651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 2996651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); 29976bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SV = llvm::ConstantVector::get(Indices); 29986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn"); 29996bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SV = Builder.CreateStore(SV, Addr); 3000651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 30016bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return SV; 30026bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } 30036bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case NEON::BI__builtin_neon_vtst_v: 30046bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case NEON::BI__builtin_neon_vtstq_v: { 3005651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3006651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 30076bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); 30086bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], 30096bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ConstantAggregateZero::get(Ty)); 30106bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return Builder.CreateSExt(Ops[0], Ty, "vtst"); 30116bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } 30126bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case NEON::BI__builtin_neon_vuzp_v: 30136bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case NEON::BI__builtin_neon_vuzpq_v: { 30146bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 30156bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 30166bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 30176bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *SV = nullptr; 3018651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 30196bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines for (unsigned vi = 0; vi != 2; ++vi) { 30206bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SmallVector<Constant*, 16> Indices; 30216bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) 30226bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Indices.push_back(ConstantInt::get(Int32Ty, 2*i+vi)); 3023651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 30246bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); 30256bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SV = llvm::ConstantVector::get(Indices); 30266bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp"); 30276bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SV = Builder.CreateStore(SV, Addr); 30286bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } 30296bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return SV; 3030651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 30316bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case NEON::BI__builtin_neon_vzip_v: 30326bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case NEON::BI__builtin_neon_vzipq_v: { 30336bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 3034651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3035651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 30366bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *SV = nullptr; 3037651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 30386bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines for (unsigned vi = 0; vi != 2; ++vi) { 30396bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SmallVector<Constant*, 16> Indices; 30406bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 30416bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Indices.push_back(ConstantInt::get(Int32Ty, (i + vi*e) >> 1)); 30426bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Indices.push_back(ConstantInt::get(Int32Ty, ((i + vi*e) >> 1)+e)); 30436bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } 30446bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); 30456bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SV = llvm::ConstantVector::get(Indices); 30466bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip"); 30476bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SV = Builder.CreateStore(SV, Addr); 30486bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } 30496bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return SV; 3050651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3051651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 30526bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 30536bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines assert(Int && "Expected valid intrinsic number"); 30546bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 30556bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // Determine the type(s) of this overloaded AArch64 intrinsic. 30566bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = LookupNeonLLVMIntrinsic(Int, Modifier, Ty, E); 30576bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 30586bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *Result = EmitNeonCall(F, Ops, NameHint); 30596bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines llvm::Type *ResultType = ConvertType(E->getType()); 30606bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // AArch64 intrinsic one-element vector type cast to 30616bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // scalar type expected by the builtin 30626bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return Builder.CreateBitCast(Result, ResultType, NameHint); 30636bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines} 30646bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 30656bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen HinesValue *CodeGenFunction::EmitAArch64CompareBuiltinExpr( 30666bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *Op, llvm::Type *Ty, const CmpInst::Predicate Fp, 30676bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines const CmpInst::Predicate Ip, const Twine &Name) { 30686bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines llvm::Type *OTy = Op->getType(); 30696bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 30706bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // FIXME: this is utterly horrific. We should not be looking at previous 30716bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // codegen context to find out what needs doing. Unfortunately TableGen 30726bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // currently gives us exactly the same calls for vceqz_f32 and vceqz_s32 30736bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // (etc). 30746bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (BitCastInst *BI = dyn_cast<BitCastInst>(Op)) 30756bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines OTy = BI->getOperand(0)->getType(); 30766bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 30776bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Op = Builder.CreateBitCast(Op, OTy); 30786bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (OTy->getScalarType()->isFloatingPointTy()) { 30796bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Op = Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy)); 30806bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } else { 30816bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Op = Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy)); 3082651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 30836bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return Builder.CreateSExt(Op, Ty, Name); 30846bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines} 30856bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 30866bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hinesstatic Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops, 30876bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *ExtOp, Value *IndexOp, 30886bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines llvm::Type *ResTy, unsigned IntID, 30896bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines const char *Name) { 30906bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SmallVector<Value *, 2> TblOps; 30916bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (ExtOp) 30926bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines TblOps.push_back(ExtOp); 30936bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 30946bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // Build a vector containing sequential number like (0, 1, 2, ..., 15) 30956bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SmallVector<Constant*, 16> Indices; 30966bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines llvm::VectorType *TblTy = cast<llvm::VectorType>(Ops[0]->getType()); 30976bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines for (unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) { 30986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Indices.push_back(ConstantInt::get(CGF.Int32Ty, 2*i)); 30996bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Indices.push_back(ConstantInt::get(CGF.Int32Ty, 2*i+1)); 3100651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 31016bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *SV = llvm::ConstantVector::get(Indices); 31026bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 31036bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines int PairPos = 0, End = Ops.size() - 1; 31046bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines while (PairPos < End) { 31056bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], 31066bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[PairPos+1], SV, Name)); 31076bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines PairPos += 2; 3108651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 31096bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 31106bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // If there's an odd number of 64-bit lookup table, fill the high 64-bit 31116bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // of the 128-bit lookup table with zero. 31126bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (PairPos == End) { 31136bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *ZeroTbl = ConstantAggregateZero::get(TblTy); 31146bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], 31156bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ZeroTbl, SV, Name)); 3116651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 31176bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 31186bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *TblF; 31196bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines TblOps.push_back(IndexOp); 31206bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines TblF = CGF.CGM.getIntrinsic(IntID, ResTy); 31216bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 31226bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return CGF.EmitNeonCall(TblF, TblOps, Name); 3123651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 3124651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3125651f13cea278ec967336033dd032faef0e9fc2ecStephen HinesValue *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, 3126651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const CallExpr *E) { 31276bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines unsigned HintID = static_cast<unsigned>(-1); 31286bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines switch (BuiltinID) { 31296bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines default: break; 3130176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case ARM::BI__builtin_arm_nop: 3131176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines HintID = 0; 3132176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines break; 3133c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case ARM::BI__builtin_arm_yield: 31346bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case ARM::BI__yield: 31356bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines HintID = 1; 31366bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines break; 3137c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case ARM::BI__builtin_arm_wfe: 31386bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case ARM::BI__wfe: 31396bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines HintID = 2; 31406bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines break; 3141c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case ARM::BI__builtin_arm_wfi: 31426bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case ARM::BI__wfi: 31436bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines HintID = 3; 31446bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines break; 3145c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case ARM::BI__builtin_arm_sev: 31466bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case ARM::BI__sev: 31476bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines HintID = 4; 31486bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines break; 3149c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case ARM::BI__builtin_arm_sevl: 31506bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case ARM::BI__sevl: 31516bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines HintID = 5; 31526bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines break; 31536bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } 31546bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 31556bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (HintID != static_cast<unsigned>(-1)) { 31566bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::arm_hint); 31576bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID)); 31586bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } 31596bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 3160176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (BuiltinID == ARM::BI__builtin_arm_dbg) { 3161176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *Option = EmitScalarExpr(E->getArg(0)); 3162176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_dbg), Option); 3163176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 3164176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 3165176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (BuiltinID == ARM::BI__builtin_arm_prefetch) { 3166176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *Address = EmitScalarExpr(E->getArg(0)); 3167176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *RW = EmitScalarExpr(E->getArg(1)); 3168176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *IsData = EmitScalarExpr(E->getArg(2)); 3169176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 3170176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // Locality is not supported on ARM target 3171176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *Locality = llvm::ConstantInt::get(Int32Ty, 3); 3172176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 3173176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *F = CGM.getIntrinsic(Intrinsic::prefetch); 3174176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return Builder.CreateCall4(F, Address, RW, Locality, IsData); 3175176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 3176176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 3177c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines if (BuiltinID == ARM::BI__builtin_arm_rbit) { 3178c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_rbit), 3179c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines EmitScalarExpr(E->getArg(0)), 3180c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines "rbit"); 3181c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines } 3182c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 3183651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (BuiltinID == ARM::BI__clear_cache) { 3184651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments"); 3185651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const FunctionDecl *FD = E->getDirectCallee(); 3186651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value*, 2> Ops; 3187651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines for (unsigned i = 0; i < 2; i++) 3188651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(i))); 3189651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType()); 3190651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty); 3191651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines StringRef Name = FD->getName(); 3192651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); 3193651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3194651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3195651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (BuiltinID == ARM::BI__builtin_arm_ldrexd || 3196c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines ((BuiltinID == ARM::BI__builtin_arm_ldrex || 3197c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines BuiltinID == ARM::BI__builtin_arm_ldaex) && 3198c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines getContext().getTypeSize(E->getType()) == 64) || 3199c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines BuiltinID == ARM::BI__ldrexd) { 3200c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Function *F; 3201c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 3202c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines switch (BuiltinID) { 3203c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines default: llvm_unreachable("unexpected builtin"); 3204c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case ARM::BI__builtin_arm_ldaex: 3205c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines F = CGM.getIntrinsic(Intrinsic::arm_ldaexd); 3206c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines break; 3207c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case ARM::BI__builtin_arm_ldrexd: 3208c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case ARM::BI__builtin_arm_ldrex: 3209c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case ARM::BI__ldrexd: 3210c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines F = CGM.getIntrinsic(Intrinsic::arm_ldrexd); 3211c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines break; 3212c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines } 3213651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3214651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *LdPtr = EmitScalarExpr(E->getArg(0)); 3215651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy), 3216651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "ldrexd"); 3217651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3218651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Val0 = Builder.CreateExtractValue(Val, 1); 3219651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Val1 = Builder.CreateExtractValue(Val, 0); 3220651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val0 = Builder.CreateZExt(Val0, Int64Ty); 3221651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val1 = Builder.CreateZExt(Val1, Int64Ty); 3222651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3223651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *ShiftCst = llvm::ConstantInt::get(Int64Ty, 32); 3224651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */); 3225651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val = Builder.CreateOr(Val, Val1); 3226651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateBitCast(Val, ConvertType(E->getType())); 3227651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3228651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3229c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines if (BuiltinID == ARM::BI__builtin_arm_ldrex || 3230c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines BuiltinID == ARM::BI__builtin_arm_ldaex) { 3231651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *LoadAddr = EmitScalarExpr(E->getArg(0)); 3232651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3233651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines QualType Ty = E->getType(); 3234651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *RealResTy = ConvertType(Ty); 3235651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *IntResTy = llvm::IntegerType::get(getLLVMContext(), 3236651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines getContext().getTypeSize(Ty)); 3237651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines LoadAddr = Builder.CreateBitCast(LoadAddr, IntResTy->getPointerTo()); 3238651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3239c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_ldaex 3240c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines ? Intrinsic::arm_ldaex 3241c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines : Intrinsic::arm_ldrex, 3242c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines LoadAddr->getType()); 3243651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Val = Builder.CreateCall(F, LoadAddr, "ldrex"); 3244651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3245651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (RealResTy->isPointerTy()) 3246651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateIntToPtr(Val, RealResTy); 3247651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines else { 3248651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val = Builder.CreateTruncOrBitCast(Val, IntResTy); 3249651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateBitCast(Val, RealResTy); 3250651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3251651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3252651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3253651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (BuiltinID == ARM::BI__builtin_arm_strexd || 3254c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines ((BuiltinID == ARM::BI__builtin_arm_stlex || 3255c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines BuiltinID == ARM::BI__builtin_arm_strex) && 3256651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines getContext().getTypeSize(E->getArg(0)->getType()) == 64)) { 3257c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex 3258c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines ? Intrinsic::arm_stlexd 3259c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines : Intrinsic::arm_strexd); 3260651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, NULL); 3261651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3262651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Tmp = CreateMemTemp(E->getArg(0)->getType()); 3263651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Val = EmitScalarExpr(E->getArg(0)); 3264651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Builder.CreateStore(Val, Tmp); 3265651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3266651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy)); 3267651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val = Builder.CreateLoad(LdPtr); 3268651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3269651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Arg0 = Builder.CreateExtractValue(Val, 0); 3270651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Arg1 = Builder.CreateExtractValue(Val, 1); 3271651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), Int8PtrTy); 3272651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall3(F, Arg0, Arg1, StPtr, "strexd"); 32732e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu } 3274651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3275c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines if (BuiltinID == ARM::BI__builtin_arm_strex || 3276c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines BuiltinID == ARM::BI__builtin_arm_stlex) { 3277651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *StoreVal = EmitScalarExpr(E->getArg(0)); 3278651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *StoreAddr = EmitScalarExpr(E->getArg(1)); 3279651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3280651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines QualType Ty = E->getArg(0)->getType(); 3281651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(), 3282651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines getContext().getTypeSize(Ty)); 3283651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo()); 3284651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3285651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (StoreVal->getType()->isPointerTy()) 3286651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines StoreVal = Builder.CreatePtrToInt(StoreVal, Int32Ty); 3287651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines else { 3288651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines StoreVal = Builder.CreateBitCast(StoreVal, StoreTy); 3289651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int32Ty); 3290651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3291651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3292c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex 3293c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines ? Intrinsic::arm_stlex 3294c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines : Intrinsic::arm_strex, 3295c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines StoreAddr->getType()); 3296651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall2(F, StoreVal, StoreAddr, "strex"); 32972e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu } 3298651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3299651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (BuiltinID == ARM::BI__builtin_arm_clrex) { 3300651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(Intrinsic::arm_clrex); 3301651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall(F); 33022e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu } 33032e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu 3304651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // CRC32 3305651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic; 3306651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 3307651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case ARM::BI__builtin_arm_crc32b: 3308651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CRCIntrinsicID = Intrinsic::arm_crc32b; break; 3309651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case ARM::BI__builtin_arm_crc32cb: 3310651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CRCIntrinsicID = Intrinsic::arm_crc32cb; break; 3311651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case ARM::BI__builtin_arm_crc32h: 3312651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CRCIntrinsicID = Intrinsic::arm_crc32h; break; 3313651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case ARM::BI__builtin_arm_crc32ch: 3314651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CRCIntrinsicID = Intrinsic::arm_crc32ch; break; 3315651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case ARM::BI__builtin_arm_crc32w: 3316651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case ARM::BI__builtin_arm_crc32d: 3317651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CRCIntrinsicID = Intrinsic::arm_crc32w; break; 3318651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case ARM::BI__builtin_arm_crc32cw: 3319651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case ARM::BI__builtin_arm_crc32cd: 3320651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CRCIntrinsicID = Intrinsic::arm_crc32cw; break; 3321651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 33222e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu 3323651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (CRCIntrinsicID != Intrinsic::not_intrinsic) { 3324651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Arg0 = EmitScalarExpr(E->getArg(0)); 3325651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Arg1 = EmitScalarExpr(E->getArg(1)); 3326aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu 3327651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // crc32{c,}d intrinsics are implemnted as two calls to crc32{c,}w 3328651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // intrinsics, hence we need different codegen for these cases. 3329651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (BuiltinID == ARM::BI__builtin_arm_crc32d || 3330651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines BuiltinID == ARM::BI__builtin_arm_crc32cd) { 3331651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *C1 = llvm::ConstantInt::get(Int64Ty, 32); 3332651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Arg1a = Builder.CreateTruncOrBitCast(Arg1, Int32Ty); 3333651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Arg1b = Builder.CreateLShr(Arg1, C1); 3334651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Arg1b = Builder.CreateTruncOrBitCast(Arg1b, Int32Ty); 33352e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu 3336651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(CRCIntrinsicID); 3337651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Res = Builder.CreateCall2(F, Arg0, Arg1a); 3338651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall2(F, Res, Arg1b); 3339651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } else { 3340651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Arg1 = Builder.CreateZExtOrBitCast(Arg1, Int32Ty); 3341ff920eec4d449bee560d8d99636ad0eb50cd9d8dTim Northover 3342651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(CRCIntrinsicID); 3343651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall2(F, Arg0, Arg1); 3344651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3345ff920eec4d449bee560d8d99636ad0eb50cd9d8dTim Northover } 3346ff920eec4d449bee560d8d99636ad0eb50cd9d8dTim Northover 3347651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value*, 4> Ops; 33486bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines llvm::Value *Align = nullptr; 3349b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) { 3350dd12780e86575795fa912529a911b01e2abc4677Hao Liu if (i == 0) { 3351dd12780e86575795fa912529a911b01e2abc4677Hao Liu switch (BuiltinID) { 3352651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_v: 3353651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_v: 3354651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_lane_v: 3355651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_lane_v: 3356651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_dup_v: 3357651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_dup_v: 3358651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_v: 3359651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_v: 3360651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_lane_v: 3361651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_lane_v: 3362651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2_v: 3363651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2q_v: 3364651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2_lane_v: 3365651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2q_lane_v: 3366651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3_v: 3367651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3q_v: 3368651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3_lane_v: 3369651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3q_lane_v: 3370651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4_v: 3371651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4q_v: 3372651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4_lane_v: 3373651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4q_lane_v: 3374dd12780e86575795fa912529a911b01e2abc4677Hao Liu // Get the alignment for the argument in addition to the value; 3375dd12780e86575795fa912529a911b01e2abc4677Hao Liu // we'll use it later. 3376651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines std::pair<llvm::Value*, unsigned> Src = 3377dd12780e86575795fa912529a911b01e2abc4677Hao Liu EmitPointerWithAlignment(E->getArg(0)); 3378dd12780e86575795fa912529a911b01e2abc4677Hao Liu Ops.push_back(Src.first); 3379dd12780e86575795fa912529a911b01e2abc4677Hao Liu Align = Builder.getInt32(Src.second); 3380dd12780e86575795fa912529a911b01e2abc4677Hao Liu continue; 3381dd12780e86575795fa912529a911b01e2abc4677Hao Liu } 3382dd12780e86575795fa912529a911b01e2abc4677Hao Liu } 3383dd12780e86575795fa912529a911b01e2abc4677Hao Liu if (i == 1) { 3384dd12780e86575795fa912529a911b01e2abc4677Hao Liu switch (BuiltinID) { 3385651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_v: 3386651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2q_v: 3387651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_v: 3388651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3q_v: 3389651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_v: 3390651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4q_v: 3391651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_lane_v: 3392651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2q_lane_v: 3393651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_lane_v: 3394651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3q_lane_v: 3395651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_lane_v: 3396651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4q_lane_v: 3397651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_dup_v: 3398651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_dup_v: 3399651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_dup_v: 3400dd12780e86575795fa912529a911b01e2abc4677Hao Liu // Get the alignment for the argument in addition to the value; 3401dd12780e86575795fa912529a911b01e2abc4677Hao Liu // we'll use it later. 3402651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines std::pair<llvm::Value*, unsigned> Src = 3403dd12780e86575795fa912529a911b01e2abc4677Hao Liu EmitPointerWithAlignment(E->getArg(1)); 3404dd12780e86575795fa912529a911b01e2abc4677Hao Liu Ops.push_back(Src.first); 3405dd12780e86575795fa912529a911b01e2abc4677Hao Liu Align = Builder.getInt32(Src.second); 3406dd12780e86575795fa912529a911b01e2abc4677Hao Liu continue; 3407dd12780e86575795fa912529a911b01e2abc4677Hao Liu } 3408dd12780e86575795fa912529a911b01e2abc4677Hao Liu } 3409b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover Ops.push_back(EmitScalarExpr(E->getArg(i))); 3410b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover } 3411b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover 3412651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 3413651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines default: break; 3414651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // vget_lane and vset_lane are not overloaded and do not have an extra 3415651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // argument that specifies the vector type. 3416651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_i8: 3417651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_i16: 3418651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_i32: 3419651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_i64: 3420651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_f32: 3421651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_i8: 3422651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_i16: 3423651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_i32: 3424651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_i64: 3425651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_f32: 3426651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 3427651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vget_lane"); 3428651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_i8: 3429651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_i16: 3430651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_i32: 3431651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_i64: 3432651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_f32: 3433651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_i8: 3434651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_i16: 3435651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_i32: 3436651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_i64: 3437651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_f32: 3438651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(2))); 3439651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 3440651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3441651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Non-polymorphic crypto instructions also not overloaded 3442651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsha1h_u32: 3443651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 3444651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops, 3445651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vsha1h"); 3446651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsha1cq_u32: 3447651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(2))); 3448651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops, 3449651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vsha1h"); 3450651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsha1pq_u32: 3451651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(2))); 3452651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops, 3453651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vsha1h"); 3454651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsha1mq_u32: 3455651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(2))); 3456651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops, 3457651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vsha1h"); 3458651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3459651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3460b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover // Get the last argument, which specifies the vector type. 3461b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover llvm::APSInt Result; 3462651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const Expr *Arg = E->getArg(E->getNumArgs()-1); 3463b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover if (!Arg->isIntegerConstantExpr(Result, getContext())) 34646bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 3465b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover 3466651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f || 3467651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines BuiltinID == ARM::BI__builtin_arm_vcvtr_d) { 3468651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Determine the overloaded type of this builtin. 3469651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty; 3470651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f) 3471651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = FloatTy; 3472912502b4996b14db31b498cb1eef2b17d7d66d57Hao Liu else 3473651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = DoubleTy; 3474912502b4996b14db31b498cb1eef2b17d7d66d57Hao Liu 3475651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Determine whether this is an unsigned conversion or not. 3476651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool usgn = Result.getZExtValue() == 1; 3477651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr; 3478b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover 3479651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Call the appropriate intrinsic. 3480dd12780e86575795fa912529a911b01e2abc4677Hao Liu Function *F = CGM.getIntrinsic(Int, Ty); 3481651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall(F, Ops, "vcvtr"); 3482dd12780e86575795fa912529a911b01e2abc4677Hao Liu } 3483651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3484651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Determine the type of this overloaded NEON intrinsic. 3485651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags Type(Result.getZExtValue()); 3486651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool usgn = Type.isUnsigned(); 3487651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool rightShift = false; 3488651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3489651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType *VTy = GetNeonType(this, Type); 3490651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty = VTy; 3491651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (!Ty) 34926bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 3493651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3494651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Many NEON builtins have identical semantics and uses in ARM and 3495651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // AArch64. Emit these in a single function. 3496176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto IntrinsicMap = makeArrayRef(ARMSIMDIntrinsicMap); 3497651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap( 3498651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines IntrinsicMap, BuiltinID, NEONSIMDIntrinsicsProvenSorted); 3499651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Builtin) 3500651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitCommonNeonBuiltinExpr( 3501651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic, 3502651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Builtin->NameHint, Builtin->TypeModifier, E, Ops, Align); 3503651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3504651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Int; 3505651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 35066bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines default: return nullptr; 3507651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_lane_v: 3508651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Handle 64-bit integer elements as a special case. Use shuffles of 3509651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // one-element vectors to avoid poor code for i64 in the backend. 3510651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (VTy->getElementType()->isIntegerTy(64)) { 3511651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Extract the other lane. 3512651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3513651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines int Lane = cast<ConstantInt>(Ops[2])->getZExtValue(); 3514651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *SV = llvm::ConstantVector::get(ConstantInt::get(Int32Ty, 1-Lane)); 3515651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); 3516651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Load the value as a one-element vector. 3517651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::VectorType::get(VTy->getElementType(), 1); 3518651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Ty); 3519651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Ld = Builder.CreateCall2(F, Ops[0], Align); 3520651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Combine them. 3521651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Constant*, 2> Indices; 3522651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Indices.push_back(ConstantInt::get(Int32Ty, 1-Lane)); 3523651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Indices.push_back(ConstantInt::get(Int32Ty, Lane)); 3524651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SV = llvm::ConstantVector::get(Indices); 3525651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateShuffleVector(Ops[1], Ld, SV, "vld1q_lane"); 3526dd12780e86575795fa912529a911b01e2abc4677Hao Liu } 3527651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // fall through 3528651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_lane_v: { 352911a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 353011a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ty = llvm::PointerType::getUnqual(VTy->getElementType()); 353111a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 353211a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu LoadInst *Ld = Builder.CreateLoad(Ops[0]); 353311a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ld->setAlignment(cast<ConstantInt>(Align)->getZExtValue()); 353411a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane"); 353511a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu } 3536651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_dup_v: 3537651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_dup_v: 3538651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_dup_v: { 3539651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Handle 64-bit elements as a special-case. There is no "dup" needed. 3540651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (VTy->getElementType()->getPrimitiveSizeInBits() == 64) { 354111a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu switch (BuiltinID) { 3542651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_dup_v: 354311a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Int = Intrinsic::arm_neon_vld2; 354411a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu break; 3545651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_dup_v: 354611a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Int = Intrinsic::arm_neon_vld3; 354711a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu break; 3548651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_dup_v: 354911a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Int = Intrinsic::arm_neon_vld4; 355011a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu break; 3551651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines default: llvm_unreachable("unknown vld_dup intrinsic?"); 355211a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu } 355311a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Function *F = CGM.getIntrinsic(Int, Ty); 355411a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld_dup"); 355511a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 355611a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 355711a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu return Builder.CreateStore(Ops[1], Ops[0]); 355811a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu } 355911a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu switch (BuiltinID) { 3560651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_dup_v: 356111a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Int = Intrinsic::arm_neon_vld2lane; 356211a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu break; 3563651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_dup_v: 356411a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Int = Intrinsic::arm_neon_vld3lane; 356511a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu break; 3566651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_dup_v: 356711a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Int = Intrinsic::arm_neon_vld4lane; 356811a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu break; 3569651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines default: llvm_unreachable("unknown vld_dup intrinsic?"); 357011a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu } 357111a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Function *F = CGM.getIntrinsic(Int, Ty); 357211a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu llvm::StructType *STy = cast<llvm::StructType>(F->getReturnType()); 357311a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu 3574651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value*, 6> Args; 357511a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Args.push_back(Ops[1]); 357611a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Args.append(STy->getNumElements(), UndefValue::get(Ty)); 357711a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu 357811a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu llvm::Constant *CI = ConstantInt::get(Int32Ty, 0); 357911a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Args.push_back(CI); 358011a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Args.push_back(Align); 358111a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu 358211a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ops[1] = Builder.CreateCall(F, Args, "vld_dup"); 358311a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu // splat lane 0 to all elts in each vector of the result. 358411a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { 358511a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Value *Val = Builder.CreateExtractValue(Ops[1], i); 358611a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Value *Elt = Builder.CreateBitCast(Val, Ty); 358711a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Elt = EmitNeonSplat(Elt, CI); 358811a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Elt = Builder.CreateBitCast(Elt, Val->getType()); 358911a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ops[1] = Builder.CreateInsertValue(Ops[1], Elt, i); 359011a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu } 359111a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 359211a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 359311a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu return Builder.CreateStore(Ops[1], Ops[0]); 359411a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu } 3595651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqrshrn_n_v: 3596651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = 3597651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns; 3598651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n", 3599651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1, true); 3600651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqrshrun_n_v: 3601651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqrshiftnsu, Ty), 3602651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vqrshrun_n", 1, true); 3603651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshrn_n_v: 3604651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns; 3605651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n", 3606651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1, true); 3607651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshrun_n_v: 3608651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftnsu, Ty), 3609651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vqshrun_n", 1, true); 3610651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrecpe_v: 3611651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrecpeq_v: 3612651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrecpe, Ty), 3613651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vrecpe"); 3614651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrshrn_n_v: 3615651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrshiftn, Ty), 3616651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vrshrn_n", 1, true); 3617651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrsra_n_v: 3618651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrsraq_n_v: 36190aa1a88e19235574481e46e9e6e9ce66a9e6624fJiangning Liu Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 36200aa1a88e19235574481e46e9e6e9ce66a9e6624fJiangning Liu Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3621651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true); 3622651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts; 3623651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall2(CGM.getIntrinsic(Int, Ty), Ops[1], Ops[2]); 3624651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n"); 3625651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsri_n_v: 3626651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsriq_n_v: 3627651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines rightShift = true; 3628651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsli_n_v: 3629651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsliq_n_v: 3630651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift); 3631651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftins, Ty), 3632651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vsli_n"); 3633651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsra_n_v: 3634651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsraq_n_v: 36350aa1a88e19235574481e46e9e6e9ce66a9e6624fJiangning Liu Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3636651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); 3637651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Ops[0], Ops[1]); 3638651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_lane_v: 3639651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Handle 64-bit integer elements as a special case. Use a shuffle to get 3640651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // a one-element vector and avoid poor code for i64 in the backend. 3641651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (VTy->getElementType()->isIntegerTy(64)) { 3642651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3643651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2])); 3644651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); 3645651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Align; 3646651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1, 3647651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1]->getType()), Ops); 364851cc0172a173599b769968696e20638754d1dcd6Ana Pazos } 3649651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // fall through 3650651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_lane_v: { 3651b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3652651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); 3653651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 3654651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines StoreInst *St = Builder.CreateStore(Ops[1], 3655651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Builder.CreateBitCast(Ops[0], Ty)); 3656651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines St->setAlignment(cast<ConstantInt>(Align)->getZExtValue()); 3657651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return St; 36588137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 3659651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl1_v: 3660651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl1), 3661651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbl1"); 3662651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl2_v: 3663651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl2), 3664651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbl2"); 3665651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl3_v: 3666651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl3), 3667651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbl3"); 3668651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl4_v: 3669651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl4), 3670651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbl4"); 3671651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx1_v: 3672651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx1), 3673651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbx1"); 3674651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx2_v: 3675651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx2), 3676651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbx2"); 3677651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx3_v: 3678651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx3), 3679651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbx3"); 3680651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx4_v: 3681651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx4), 3682651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbx4"); 36838137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 3684651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 3685651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 36866bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hinesstatic Value *EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID, 3687651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const CallExpr *E, 3688651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVectorImpl<Value *> &Ops) { 3689651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned int Int = 0; 36906bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines const char *s = nullptr; 3691651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3692651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 3693651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines default: 36946bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 3695651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl1_v: 3696651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl1_v: 3697651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl1q_v: 3698651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl2_v: 3699651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl2_v: 3700651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl2q_v: 3701651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl3_v: 3702651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl3_v: 3703651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl3q_v: 3704651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl4_v: 3705651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl4_v: 3706651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl4q_v: 3707651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 3708651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx1_v: 3709651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx1_v: 3710651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx1q_v: 3711651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx2_v: 3712651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx2_v: 3713651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx2q_v: 3714651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx3_v: 3715651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx3_v: 3716651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx3q_v: 3717651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx4_v: 3718651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx4_v: 3719651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx4q_v: 3720651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 37218137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 3722651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3723651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines assert(E->getNumArgs() >= 3); 3724651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3725651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Get the last argument, which specifies the vector type. 3726651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::APSInt Result; 3727651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const Expr *Arg = E->getArg(E->getNumArgs() - 1); 3728651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (!Arg->isIntegerConstantExpr(Result, CGF.getContext())) 37296bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 3730651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3731651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Determine the type of this overloaded NEON intrinsic. 3732651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags Type(Result.getZExtValue()); 3733651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType *VTy = GetNeonType(&CGF, Type); 3734651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty = VTy; 3735651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (!Ty) 37366bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 3737651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3738651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned nElts = VTy->getNumElements(); 3739651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3740651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CodeGen::CGBuilderTy &Builder = CGF.Builder; 3741651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3742651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // AArch64 scalar builtins are not overloaded, they do not have an extra 3743651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // argument that specifies the vector type, need to handle each case. 3744651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value *, 2> TblOps; 3745651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 3746651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl1_v: { 3747651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[0]); 37486bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return packTBLDVectorList(CGF, TblOps, nullptr, Ops[1], Ty, 37496bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Intrinsic::aarch64_neon_tbl1, "vtbl1"); 37508137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 3751651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl2_v: { 3752651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[0]); 3753651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[1]); 37546bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return packTBLDVectorList(CGF, TblOps, nullptr, Ops[2], Ty, 37556bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Intrinsic::aarch64_neon_tbl1, "vtbl1"); 37568137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 3757651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl3_v: { 3758651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[0]); 3759651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[1]); 3760651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[2]); 37616bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return packTBLDVectorList(CGF, TblOps, nullptr, Ops[3], Ty, 37626bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Intrinsic::aarch64_neon_tbl2, "vtbl2"); 37638137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 3764651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl4_v: { 3765651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[0]); 3766651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[1]); 3767651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[2]); 3768651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[3]); 37696bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return packTBLDVectorList(CGF, TblOps, nullptr, Ops[4], Ty, 37706bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Intrinsic::aarch64_neon_tbl2, "vtbl2"); 37718137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 3772651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx1_v: { 3773651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[1]); 37746bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *TblRes = packTBLDVectorList(CGF, TblOps, nullptr, Ops[2], Ty, 37756bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Intrinsic::aarch64_neon_tbl1, "vtbl1"); 3776651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3777651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Constant *Eight = ConstantInt::get(VTy->getElementType(), 8); 3778651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value* EightV = llvm::ConstantVector::getSplat(nElts, Eight); 3779651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV); 3780651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CmpRes = Builder.CreateSExt(CmpRes, Ty); 3781651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3782651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); 3783651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes); 3784651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx"); 37858137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 3786651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx2_v: { 3787651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[1]); 3788651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[2]); 3789651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return packTBLDVectorList(CGF, TblOps, Ops[0], Ops[3], Ty, 37906bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Intrinsic::aarch64_neon_tbx1, "vtbx1"); 37918137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 3792651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx3_v: { 3793651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[1]); 3794651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[2]); 3795651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[3]); 37966bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *TblRes = packTBLDVectorList(CGF, TblOps, nullptr, Ops[4], Ty, 37976bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Intrinsic::aarch64_neon_tbl2, "vtbl2"); 3798651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3799651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Constant *TwentyFour = ConstantInt::get(VTy->getElementType(), 24); 3800651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value* TwentyFourV = llvm::ConstantVector::getSplat(nElts, TwentyFour); 3801651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4], 3802651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TwentyFourV); 3803651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CmpRes = Builder.CreateSExt(CmpRes, Ty); 3804651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3805651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); 3806651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes); 3807651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx"); 38088137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 3809651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx4_v: { 3810651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[1]); 3811651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[2]); 3812651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[3]); 3813651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TblOps.push_back(Ops[4]); 3814651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return packTBLDVectorList(CGF, TblOps, Ops[0], Ops[5], Ty, 38156bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Intrinsic::aarch64_neon_tbx2, "vtbx2"); 3816651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3817651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl1_v: 3818651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl1q_v: 38196bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_tbl1; s = "vtbl1"; break; 3820651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl2_v: 3821651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl2q_v: { 38226bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_tbl2; s = "vtbl2"; break; 3823651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl3_v: 3824651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl3q_v: 38256bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_tbl3; s = "vtbl3"; break; 3826651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl4_v: 3827651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl4q_v: 38286bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_tbl4; s = "vtbl4"; break; 3829651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx1_v: 3830651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx1q_v: 38316bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_tbx1; s = "vtbx1"; break; 3832651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx2_v: 3833651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx2q_v: 38346bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_tbx2; s = "vtbx2"; break; 3835651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx3_v: 3836651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx3q_v: 38376bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_tbx3; s = "vtbx3"; break; 3838651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx4_v: 3839651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx4q_v: 38406bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_tbx4; s = "vtbx4"; break; 38418137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 3842b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover } 3843651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3844651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (!Int) 38456bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 3846651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3847651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGF.CGM.getIntrinsic(Int, Ty); 3848651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return CGF.EmitNeonCall(F, Ops, s); 3849ff920eec4d449bee560d8d99636ad0eb50cd9d8dTim Northover} 3850ff920eec4d449bee560d8d99636ad0eb50cd9d8dTim Northover 3851651f13cea278ec967336033dd032faef0e9fc2ecStephen HinesValue *CodeGenFunction::vectorWrapScalar16(Value *Op) { 3852651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *VTy = llvm::VectorType::get(Int16Ty, 4); 3853651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Op = Builder.CreateBitCast(Op, Int16Ty); 3854651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *V = UndefValue::get(VTy); 3855c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Constant *CI = ConstantInt::get(SizeTy, 0); 3856651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Op = Builder.CreateInsertElement(V, Op, CI); 3857651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Op; 3858651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 3859651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3860651f13cea278ec967336033dd032faef0e9fc2ecStephen HinesValue *CodeGenFunction::vectorWrapScalar8(Value *Op) { 3861651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *VTy = llvm::VectorType::get(Int8Ty, 8); 3862651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Op = Builder.CreateBitCast(Op, Int8Ty); 3863651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *V = UndefValue::get(VTy); 3864c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Constant *CI = ConstantInt::get(SizeTy, 0); 3865651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Op = Builder.CreateInsertElement(V, Op, CI); 3866651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Op; 3867651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 3868651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3869651f13cea278ec967336033dd032faef0e9fc2ecStephen HinesValue *CodeGenFunction:: 3870651f13cea278ec967336033dd032faef0e9fc2ecStephen HinesemitVectorWrappedScalar8Intrinsic(unsigned Int, SmallVectorImpl<Value*> &Ops, 3871651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const char *Name) { 38726bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // i8 is not a legal types for AArch64, so we can't just use 3873c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines // a normal overloaded intrinsic call for these scalar types. Instead 3874651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // we'll build 64-bit vectors w/ lane zero being our input values and 3875651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // perform the operation on that. The back end can pattern match directly 3876651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // to the scalar instruction. 3877651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = vectorWrapScalar8(Ops[0]); 3878651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = vectorWrapScalar8(Ops[1]); 3879651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *VTy = llvm::VectorType::get(Int8Ty, 8); 3880651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *V = EmitNeonCall(CGM.getIntrinsic(Int, VTy), Ops, Name); 3881c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Constant *CI = ConstantInt::get(SizeTy, 0); 3882651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(V, CI, "lane0"); 3883651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 3884651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3885651f13cea278ec967336033dd032faef0e9fc2ecStephen HinesValue *CodeGenFunction:: 3886651f13cea278ec967336033dd032faef0e9fc2ecStephen HinesemitVectorWrappedScalar16Intrinsic(unsigned Int, SmallVectorImpl<Value*> &Ops, 3887651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const char *Name) { 38886bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // i16 is not a legal types for AArch64, so we can't just use 3889c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines // a normal overloaded intrinsic call for these scalar types. Instead 3890651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // we'll build 64-bit vectors w/ lane zero being our input values and 3891651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // perform the operation on that. The back end can pattern match directly 3892651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // to the scalar instruction. 3893651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = vectorWrapScalar16(Ops[0]); 3894651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = vectorWrapScalar16(Ops[1]); 3895651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *VTy = llvm::VectorType::get(Int16Ty, 4); 3896651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *V = EmitNeonCall(CGM.getIntrinsic(Int, VTy), Ops, Name); 3897c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Constant *CI = ConstantInt::get(SizeTy, 0); 3898651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(V, CI, "lane0"); 3899651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 3900651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 39016bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen HinesValue *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, 39026bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines const CallExpr *E) { 3903176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines unsigned HintID = static_cast<unsigned>(-1); 3904176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines switch (BuiltinID) { 3905176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines default: break; 3906176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case AArch64::BI__builtin_arm_nop: 3907176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines HintID = 0; 3908176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines break; 3909176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case AArch64::BI__builtin_arm_yield: 3910176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines HintID = 1; 3911176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines break; 3912176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case AArch64::BI__builtin_arm_wfe: 3913176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines HintID = 2; 3914176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines break; 3915176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case AArch64::BI__builtin_arm_wfi: 3916176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines HintID = 3; 3917176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines break; 3918176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case AArch64::BI__builtin_arm_sev: 3919176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines HintID = 4; 3920176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines break; 3921176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case AArch64::BI__builtin_arm_sevl: 3922176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines HintID = 5; 3923176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines break; 3924176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 3925176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 3926176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (HintID != static_cast<unsigned>(-1)) { 3927176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_hint); 3928176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID)); 3929176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 3930176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 3931176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (BuiltinID == AArch64::BI__builtin_arm_prefetch) { 3932176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *Address = EmitScalarExpr(E->getArg(0)); 3933176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *RW = EmitScalarExpr(E->getArg(1)); 3934176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *CacheLevel = EmitScalarExpr(E->getArg(2)); 3935176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *RetentionPolicy = EmitScalarExpr(E->getArg(3)); 3936176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *IsData = EmitScalarExpr(E->getArg(4)); 3937176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 3938176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *Locality = nullptr; 3939176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (cast<llvm::ConstantInt>(RetentionPolicy)->isZero()) { 3940176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // Temporal fetch, needs to convert cache level to locality. 3941176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Locality = llvm::ConstantInt::get(Int32Ty, 3942176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines -cast<llvm::ConstantInt>(CacheLevel)->getValue() + 3); 3943176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } else { 3944176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // Streaming fetch. 3945176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Locality = llvm::ConstantInt::get(Int32Ty, 0); 3946176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 3947176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 3948176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // FIXME: We need AArch64 specific LLVM intrinsic if we want to specify 3949176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // PLDL3STRM or PLDL2STRM. 3950176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *F = CGM.getIntrinsic(Intrinsic::prefetch); 3951176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return Builder.CreateCall4(F, Address, RW, Locality, IsData); 3952176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 3953176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 3954c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines if (BuiltinID == AArch64::BI__builtin_arm_rbit) { 3955c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines assert((getContext().getTypeSize(E->getType()) == 32) && 3956c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines "rbit of unusual size!"); 3957c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Arg = EmitScalarExpr(E->getArg(0)); 3958c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines return Builder.CreateCall( 3959c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines CGM.getIntrinsic(Intrinsic::aarch64_rbit, Arg->getType()), Arg, "rbit"); 3960c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines } 3961c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines if (BuiltinID == AArch64::BI__builtin_arm_rbit64) { 3962c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines assert((getContext().getTypeSize(E->getType()) == 64) && 3963c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines "rbit of unusual size!"); 3964c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Arg = EmitScalarExpr(E->getArg(0)); 3965c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines return Builder.CreateCall( 3966c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines CGM.getIntrinsic(Intrinsic::aarch64_rbit, Arg->getType()), Arg, "rbit"); 3967c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines } 3968c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 39696bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (BuiltinID == AArch64::BI__clear_cache) { 39704537d6e0f9baf2e011a4260e0d7872789b01c3f2Rafael Espindola assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments"); 397179ba509b0106fd0a1ff832baeb1fdb5430527efeRafael Espindola const FunctionDecl *FD = E->getDirectCallee(); 39725f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner SmallVector<Value*, 2> Ops; 39734537d6e0f9baf2e011a4260e0d7872789b01c3f2Rafael Espindola for (unsigned i = 0; i < 2; i++) 39748a37c79f03e62aecfa2c58b9179f90dd1fcdb253Eric Christopher Ops.push_back(EmitScalarExpr(E->getArg(i))); 39752acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType()); 39762acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty); 39775f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner StringRef Name = FD->getName(); 3978bd7370a78604e9a20d698bfe328c1e43f12a0613John McCall return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); 39792752c0137d95aa2f4ee1cdff4b564bac842e041bChris Lattner } 3980e140af3e27016f902146023fba7680b43043ec07Rafael Espindola 3981c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines if ((BuiltinID == AArch64::BI__builtin_arm_ldrex || 3982c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines BuiltinID == AArch64::BI__builtin_arm_ldaex) && 3983651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines getContext().getTypeSize(E->getType()) == 128) { 3984c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex 3985c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines ? Intrinsic::aarch64_ldaxp 3986c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines : Intrinsic::aarch64_ldxp); 398726c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes 398826c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes Value *LdPtr = EmitScalarExpr(E->getArg(0)); 398909df2b066221d869f17f4b5762405f111a65f983Tim Northover Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy), 3990651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "ldxp"); 399126c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes 399226c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes Value *Val0 = Builder.CreateExtractValue(Val, 1); 399326c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes Value *Val1 = Builder.CreateExtractValue(Val, 0); 3994651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128); 3995651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val0 = Builder.CreateZExt(Val0, Int128Ty); 3996651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val1 = Builder.CreateZExt(Val1, Int128Ty); 399726c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes 3998651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64); 399926c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */); 400009df2b066221d869f17f4b5762405f111a65f983Tim Northover Val = Builder.CreateOr(Val, Val1); 400109df2b066221d869f17f4b5762405f111a65f983Tim Northover return Builder.CreateBitCast(Val, ConvertType(E->getType())); 4002c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines } else if (BuiltinID == AArch64::BI__builtin_arm_ldrex || 4003c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines BuiltinID == AArch64::BI__builtin_arm_ldaex) { 400409df2b066221d869f17f4b5762405f111a65f983Tim Northover Value *LoadAddr = EmitScalarExpr(E->getArg(0)); 400509df2b066221d869f17f4b5762405f111a65f983Tim Northover 400609df2b066221d869f17f4b5762405f111a65f983Tim Northover QualType Ty = E->getType(); 400709df2b066221d869f17f4b5762405f111a65f983Tim Northover llvm::Type *RealResTy = ConvertType(Ty); 400809df2b066221d869f17f4b5762405f111a65f983Tim Northover llvm::Type *IntResTy = llvm::IntegerType::get(getLLVMContext(), 400909df2b066221d869f17f4b5762405f111a65f983Tim Northover getContext().getTypeSize(Ty)); 401009df2b066221d869f17f4b5762405f111a65f983Tim Northover LoadAddr = Builder.CreateBitCast(LoadAddr, IntResTy->getPointerTo()); 401109df2b066221d869f17f4b5762405f111a65f983Tim Northover 4012c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex 4013c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines ? Intrinsic::aarch64_ldaxr 4014c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines : Intrinsic::aarch64_ldxr, 4015c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines LoadAddr->getType()); 4016651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Val = Builder.CreateCall(F, LoadAddr, "ldxr"); 401709df2b066221d869f17f4b5762405f111a65f983Tim Northover 401809df2b066221d869f17f4b5762405f111a65f983Tim Northover if (RealResTy->isPointerTy()) 401909df2b066221d869f17f4b5762405f111a65f983Tim Northover return Builder.CreateIntToPtr(Val, RealResTy); 4020651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4021651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val = Builder.CreateTruncOrBitCast(Val, IntResTy); 4022651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateBitCast(Val, RealResTy); 402309df2b066221d869f17f4b5762405f111a65f983Tim Northover } 402409df2b066221d869f17f4b5762405f111a65f983Tim Northover 4025c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines if ((BuiltinID == AArch64::BI__builtin_arm_strex || 4026c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines BuiltinID == AArch64::BI__builtin_arm_stlex) && 4027651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines getContext().getTypeSize(E->getArg(0)->getType()) == 128) { 4028c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex 4029c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines ? Intrinsic::aarch64_stlxp 4030c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines : Intrinsic::aarch64_stxp); 4031651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *STy = llvm::StructType::get(Int64Ty, Int64Ty, NULL); 403226c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes 4033651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *One = llvm::ConstantInt::get(Int32Ty, 1); 4034651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Tmp = Builder.CreateAlloca(ConvertType(E->getArg(0)->getType()), 4035651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines One); 403626c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes Value *Val = EmitScalarExpr(E->getArg(0)); 403726c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes Builder.CreateStore(Val, Tmp); 403826c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes 403926c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes Value *LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy)); 404026c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes Val = Builder.CreateLoad(LdPtr); 404126c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes 404226c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes Value *Arg0 = Builder.CreateExtractValue(Val, 0); 404326c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes Value *Arg1 = Builder.CreateExtractValue(Val, 1); 4044651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), 4045651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int8PtrTy); 4046651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall3(F, Arg0, Arg1, StPtr, "stxp"); 4047c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines } else if (BuiltinID == AArch64::BI__builtin_arm_strex || 4048c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines BuiltinID == AArch64::BI__builtin_arm_stlex) { 404909df2b066221d869f17f4b5762405f111a65f983Tim Northover Value *StoreVal = EmitScalarExpr(E->getArg(0)); 405009df2b066221d869f17f4b5762405f111a65f983Tim Northover Value *StoreAddr = EmitScalarExpr(E->getArg(1)); 405109df2b066221d869f17f4b5762405f111a65f983Tim Northover 405209df2b066221d869f17f4b5762405f111a65f983Tim Northover QualType Ty = E->getArg(0)->getType(); 405309df2b066221d869f17f4b5762405f111a65f983Tim Northover llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(), 405409df2b066221d869f17f4b5762405f111a65f983Tim Northover getContext().getTypeSize(Ty)); 405509df2b066221d869f17f4b5762405f111a65f983Tim Northover StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo()); 405609df2b066221d869f17f4b5762405f111a65f983Tim Northover 405709df2b066221d869f17f4b5762405f111a65f983Tim Northover if (StoreVal->getType()->isPointerTy()) 4058651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines StoreVal = Builder.CreatePtrToInt(StoreVal, Int64Ty); 405909df2b066221d869f17f4b5762405f111a65f983Tim Northover else { 406009df2b066221d869f17f4b5762405f111a65f983Tim Northover StoreVal = Builder.CreateBitCast(StoreVal, StoreTy); 4061651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int64Ty); 406209df2b066221d869f17f4b5762405f111a65f983Tim Northover } 406309df2b066221d869f17f4b5762405f111a65f983Tim Northover 4064c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex 4065c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines ? Intrinsic::aarch64_stlxr 4066c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines : Intrinsic::aarch64_stxr, 4067c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines StoreAddr->getType()); 4068651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall2(F, StoreVal, StoreAddr, "stxr"); 406909df2b066221d869f17f4b5762405f111a65f983Tim Northover } 407009df2b066221d869f17f4b5762405f111a65f983Tim Northover 40716bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (BuiltinID == AArch64::BI__builtin_arm_clrex) { 40726bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_clrex); 4073594e193073fe36b9ecbce9a105c269e20f82b3c9Joey Gouly return Builder.CreateCall(F); 4074594e193073fe36b9ecbce9a105c269e20f82b3c9Joey Gouly } 4075594e193073fe36b9ecbce9a105c269e20f82b3c9Joey Gouly 4076520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly // CRC32 4077520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic; 4078520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly switch (BuiltinID) { 40796bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case AArch64::BI__builtin_arm_crc32b: 40806bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines CRCIntrinsicID = Intrinsic::aarch64_crc32b; break; 40816bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case AArch64::BI__builtin_arm_crc32cb: 40826bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines CRCIntrinsicID = Intrinsic::aarch64_crc32cb; break; 40836bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case AArch64::BI__builtin_arm_crc32h: 40846bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines CRCIntrinsicID = Intrinsic::aarch64_crc32h; break; 40856bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case AArch64::BI__builtin_arm_crc32ch: 40866bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines CRCIntrinsicID = Intrinsic::aarch64_crc32ch; break; 40876bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case AArch64::BI__builtin_arm_crc32w: 40886bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines CRCIntrinsicID = Intrinsic::aarch64_crc32w; break; 40896bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case AArch64::BI__builtin_arm_crc32cw: 40906bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines CRCIntrinsicID = Intrinsic::aarch64_crc32cw; break; 40916bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case AArch64::BI__builtin_arm_crc32d: 40926bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines CRCIntrinsicID = Intrinsic::aarch64_crc32x; break; 40936bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case AArch64::BI__builtin_arm_crc32cd: 40946bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines CRCIntrinsicID = Intrinsic::aarch64_crc32cx; break; 4095520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly } 4096520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly 4097520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly if (CRCIntrinsicID != Intrinsic::not_intrinsic) { 4098520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly Value *Arg0 = EmitScalarExpr(E->getArg(0)); 4099520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly Value *Arg1 = EmitScalarExpr(E->getArg(1)); 4100651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(CRCIntrinsicID); 4101520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly 4102651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *DataTy = F->getFunctionType()->getParamType(1); 4103651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Arg1 = Builder.CreateZExtOrBitCast(Arg1, DataTy); 4104520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly 4105651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall2(F, Arg0, Arg1); 4106520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly } 4107520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly 4108651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::SmallVector<Value*, 4> Ops; 4109651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) 4110e140af3e27016f902146023fba7680b43043ec07Rafael Espindola Ops.push_back(EmitScalarExpr(E->getArg(i))); 4111651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4112176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto SISDMap = makeArrayRef(AArch64SISDIntrinsicMap); 4113651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap( 41146bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SISDMap, BuiltinID, AArch64SISDIntrinsicsProvenSorted); 4115651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4116651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Builtin) { 4117651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(E->getNumArgs() - 1))); 4118651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Result = EmitCommonNeonSISDBuiltinExpr(*this, *Builtin, Ops, E); 4119651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines assert(Result && "SISD intrinsic should have been handled"); 4120651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Result; 4121ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman } 4122e140af3e27016f902146023fba7680b43043ec07Rafael Espindola 4123651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::APSInt Result; 4124651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const Expr *Arg = E->getArg(E->getNumArgs()-1); 4125651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags Type(0); 4126651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Arg->isIntegerConstantExpr(Result, getContext())) 4127651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Determine the type of this overloaded NEON intrinsic. 4128651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Type = NeonTypeFlags(Result.getZExtValue()); 4129651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4130651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool usgn = Type.isUnsigned(); 4131651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool quad = Type.isQuad(); 4132651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4133651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Handle non-overloaded intrinsics first. 413483bbba1fc98dd3b2a351013a40808e995e648f40Bob Wilson switch (BuiltinID) { 413583bbba1fc98dd3b2a351013a40808e995e648f40Bob Wilson default: break; 4136651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vldrq_p128: { 4137651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128); 4138651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Ptr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), Int128PTy); 4139651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateLoad(Ptr); 4140651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4141651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vstrq_p128: { 4142651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128); 4143651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy); 4144651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateStore(EmitScalarExpr(E->getArg(1)), Ptr); 4145651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4146651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvts_u32_f32: 4147651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtd_u64_f64: 4148651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines usgn = true; 4149651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FALL THROUGH 4150651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvts_s32_f32: 4151651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtd_s64_f64: { 4152651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4153651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64; 4154651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty; 4155651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *FTy = Is64 ? DoubleTy : FloatTy; 4156651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], FTy); 4157651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (usgn) 4158651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFPToUI(Ops[0], InTy); 4159651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFPToSI(Ops[0], InTy); 4160651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4161651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvts_f32_u32: 4162651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtd_f64_u64: 4163651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines usgn = true; 4164651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FALL THROUGH 4165651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvts_f32_s32: 4166651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtd_f64_s64: { 4167651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4168651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64; 4169651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty; 4170651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *FTy = Is64 ? DoubleTy : FloatTy; 4171651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], InTy); 4172651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (usgn) 4173651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateUIToFP(Ops[0], FTy); 4174651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateSIToFP(Ops[0], FTy); 4175651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4176651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpaddd_s64: { 4177651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty = 4178651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::Type::getInt64Ty(getLLVMContext()), 2); 4179651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Vec = EmitScalarExpr(E->getArg(0)); 4180651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The vector is v2f64, so make sure it's bitcast to that. 4181651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Vec = Builder.CreateBitCast(Vec, Ty, "v2i64"); 4182c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0); 4183c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1); 4184651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0"); 4185651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1"); 4186651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Pairwise addition of a v2f64 into a scalar f64. 4187651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Op0, Op1, "vpaddd"); 4188651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4189651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpaddd_f64: { 4190651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty = 4191651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 2); 4192651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Vec = EmitScalarExpr(E->getArg(0)); 4193651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The vector is v2f64, so make sure it's bitcast to that. 4194651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Vec = Builder.CreateBitCast(Vec, Ty, "v2f64"); 4195c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0); 4196c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1); 4197651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0"); 4198651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1"); 4199651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Pairwise addition of a v2f64 into a scalar f64. 4200651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFAdd(Op0, Op1, "vpaddd"); 4201651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4202651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpadds_f32: { 4203651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty = 4204651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 2); 4205651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Vec = EmitScalarExpr(E->getArg(0)); 4206651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The vector is v2f32, so make sure it's bitcast to that. 4207651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Vec = Builder.CreateBitCast(Vec, Ty, "v2f32"); 4208c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0); 4209c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1); 4210651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0"); 4211651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1"); 4212651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Pairwise addition of a v2f32 into a scalar f32. 4213651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFAdd(Op0, Op1, "vpaddd"); 4214651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4215651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqzd_s64: 4216651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqzd_f64: 4217651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqzs_f32: 4218651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4219651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr( 4220651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OEQ, 4221651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ICmpInst::ICMP_EQ, "vceqz"); 4222651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgezd_s64: 4223651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgezd_f64: 4224651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgezs_f32: 4225651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4226651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr( 4227651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OGE, 4228651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ICmpInst::ICMP_SGE, "vcgez"); 4229651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclezd_s64: 4230651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclezd_f64: 4231651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclezs_f32: 4232651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4233651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr( 4234651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OLE, 4235651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ICmpInst::ICMP_SLE, "vclez"); 4236651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtzd_s64: 4237651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtzd_f64: 4238651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtzs_f32: 4239651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4240651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr( 4241651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OGT, 4242651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ICmpInst::ICMP_SGT, "vcgtz"); 4243651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltzd_s64: 4244651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltzd_f64: 4245651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltzs_f32: 4246651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4247651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr( 4248651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OLT, 4249651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ICmpInst::ICMP_SLT, "vcltz"); 4250651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4251651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqzd_u64: { 4252651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty = llvm::Type::getInt64Ty(getLLVMContext()); 4253651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 4254651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4255651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateICmp(llvm::ICmpInst::ICMP_EQ, Ops[0], 4256651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Constant::getNullValue(Ty)); 4257651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateSExt(Ops[0], Ty, "vceqzd"); 4258651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4259651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqd_f64: 4260651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_f64: 4261651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_f64: 4262651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcged_f64: 4263651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtd_f64: { 4264651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::CmpInst::Predicate P; 4265651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 4266651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines default: llvm_unreachable("missing builtin ID in switch!"); 4267651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqd_f64: P = llvm::FCmpInst::FCMP_OEQ; break; 4268651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_f64: P = llvm::FCmpInst::FCMP_OLE; break; 4269651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_f64: P = llvm::FCmpInst::FCMP_OLT; break; 4270651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcged_f64: P = llvm::FCmpInst::FCMP_OGE; break; 4271651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtd_f64: P = llvm::FCmpInst::FCMP_OGT; break; 4272651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4273651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 4274651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 4275651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); 4276651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); 4277651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd"); 4278651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4279651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqs_f32: 4280651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcles_f32: 4281651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclts_f32: 4282651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcges_f32: 4283651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgts_f32: { 4284651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::CmpInst::Predicate P; 4285651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 4286651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines default: llvm_unreachable("missing builtin ID in switch!"); 4287651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqs_f32: P = llvm::FCmpInst::FCMP_OEQ; break; 4288651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcles_f32: P = llvm::FCmpInst::FCMP_OLE; break; 4289651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclts_f32: P = llvm::FCmpInst::FCMP_OLT; break; 4290651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcges_f32: P = llvm::FCmpInst::FCMP_OGE; break; 4291651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgts_f32: P = llvm::FCmpInst::FCMP_OGT; break; 4292651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4293651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 4294651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy); 4295651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy); 4296651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); 4297651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd"); 4298651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4299651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqd_s64: 4300651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqd_u64: 4301651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtd_s64: 4302651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtd_u64: 4303651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_s64: 4304651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_u64: 4305651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcged_u64: 4306651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcged_s64: 4307651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_u64: 4308651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_s64: { 4309651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::CmpInst::Predicate P; 4310651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 4311651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines default: llvm_unreachable("missing builtin ID in switch!"); 4312651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqd_s64: 4313651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqd_u64:P = llvm::ICmpInst::ICMP_EQ;break; 4314651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtd_s64:P = llvm::ICmpInst::ICMP_SGT;break; 4315651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtd_u64:P = llvm::ICmpInst::ICMP_UGT;break; 4316651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_s64:P = llvm::ICmpInst::ICMP_SLT;break; 4317651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_u64:P = llvm::ICmpInst::ICMP_ULT;break; 4318651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcged_u64:P = llvm::ICmpInst::ICMP_UGE;break; 4319651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcged_s64:P = llvm::ICmpInst::ICMP_SGE;break; 4320651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_u64:P = llvm::ICmpInst::ICMP_ULE;break; 4321651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_s64:P = llvm::ICmpInst::ICMP_SLE;break; 4322651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4323651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 4324651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); 4325651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); 4326651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]); 4327651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateSExt(Ops[0], Int64Ty, "vceqd"); 4328651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4329651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtstd_s64: 4330651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtstd_u64: { 4331651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty = llvm::Type::getInt64Ty(getLLVMContext()); 4332651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 4333651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4334651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 4335651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); 4336651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], 4337651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Constant::getNullValue(Ty)); 4338651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateSExt(Ops[0], Ty, "vtstd"); 4339651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4340651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_i8: 4341651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_i16: 4342651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_i32: 4343651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_i64: 4344651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_f32: 4345651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_i8: 4346651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_i16: 4347651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_i32: 4348651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_i64: 4349651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_f32: 4350651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(2))); 4351651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 4352651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_f64: 4353651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The vector type needs a cast for the v1f64 variant. 4354651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], 4355651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(DoubleTy, 1)); 4356651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(2))); 4357651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 4358651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_f64: 4359651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The vector type needs a cast for the v2f64 variant. 4360651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], 4361651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 2)); 436283bbba1fc98dd3b2a351013a40808e995e648f40Bob Wilson Ops.push_back(EmitScalarExpr(E->getArg(2))); 436383bbba1fc98dd3b2a351013a40808e995e648f40Bob Wilson return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 436499c40bb13b523d58d7aeb6446e4f486d6918ca58Nate Begeman 4365651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_i8: 4366651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdupb_lane_i8: 4367651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 4368651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8)); 4369651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4370651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vget_lane"); 4371651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_i8: 4372651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdupb_laneq_i8: 4373651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 4374651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16)); 4375651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4376651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vgetq_lane"); 4377651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_i16: 4378651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vduph_lane_i16: 4379651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 4380651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4)); 4381651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4382651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vget_lane"); 4383651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_i16: 4384651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vduph_laneq_i16: 4385651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 4386651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8)); 4387651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4388651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vgetq_lane"); 4389651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_i32: 4390651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdups_lane_i32: 4391651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast( 4392651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0], 4393651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 32), 2)); 4394651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4395651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vget_lane"); 4396651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdups_lane_f32: 4397651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 4398651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 2)); 4399651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4400651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vdups_lane"); 4401651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_i32: 4402651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdups_laneq_i32: 4403651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 4404651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 32), 4)); 4405651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4406651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vgetq_lane"); 4407651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_i64: 4408651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdupd_lane_i64: 4409651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 4410651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 64), 1)); 4411651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4412651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vget_lane"); 4413651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdupd_lane_f64: 4414651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 4415651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 1)); 4416651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4417651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vdupd_lane"); 4418651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_i64: 4419651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdupd_laneq_i64: 4420651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 4421651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 64), 2)); 4422651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4423651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vgetq_lane"); 4424651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_f32: 4425651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 4426651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 2)); 4427651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4428651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vget_lane"); 4429651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_f64: 4430651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 4431651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 1)); 4432651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4433651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vget_lane"); 4434651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_f32: 4435651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdups_laneq_f32: 4436651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 4437651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 4)); 4438651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4439651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vgetq_lane"); 4440651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_f64: 4441651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdupd_laneq_f64: 4442651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 4443651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 2)); 4444651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 4445651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vgetq_lane"); 4446651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddd_s64: 4447651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddd_u64: 4448651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->getArg(1)), "vaddd"); 4449651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsubd_s64: 4450651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsubd_u64: 4451651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateSub(Ops[0], EmitScalarExpr(E->getArg(1)), "vsubd"); 4452651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlalh_s16: 4453651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlslh_s16: { 4454651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value *, 2> ProductOps; 4455651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps.push_back(vectorWrapScalar16(Ops[1])); 4456651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps.push_back(vectorWrapScalar16(EmitScalarExpr(E->getArg(2)))); 4457651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4); 44586bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), 4459651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps, "vqdmlXl"); 4460c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Constant *CI = ConstantInt::get(SizeTy, 0); 4461651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); 4462651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4463651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16 44646bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ? Intrinsic::aarch64_neon_sqadd 44656bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines : Intrinsic::aarch64_neon_sqsub; 4466651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops, "vqdmlXl"); 4467651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4468651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshlud_n_s64: { 4469651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 4470651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); 44716bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqshlu, Int64Ty), 44726bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops, "vqshlu_n"); 4473651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4474651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshld_n_u64: 4475651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshld_n_s64: { 4476651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64 44776bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ? Intrinsic::aarch64_neon_uqshl 44786bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines : Intrinsic::aarch64_neon_sqshl; 4479651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 4480651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); 44816bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vqshl_n"); 4482651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4483651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrshrd_n_u64: 4484651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrshrd_n_s64: { 4485651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64 44866bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ? Intrinsic::aarch64_neon_urshl 44876bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines : Intrinsic::aarch64_neon_srshl; 4488651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 44896bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines int SV = cast<ConstantInt>(Ops[1])->getSExtValue(); 44906bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[1] = ConstantInt::get(Int64Ty, -SV); 44916bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vrshr_n"); 4492651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4493651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrsrad_n_u64: 4494651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrsrad_n_s64: { 4495651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64 44966bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ? Intrinsic::aarch64_neon_urshl 44976bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines : Intrinsic::aarch64_neon_srshl; 4498651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); 4499651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2)))); 4500651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall2(CGM.getIntrinsic(Int, Int64Ty), Ops[1], 4501651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Builder.CreateSExt(Ops[2], Int64Ty)); 4502651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty)); 4503651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4504651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshld_n_s64: 4505651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshld_n_u64: { 4506651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1))); 4507651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateShl( 45086bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()), "shld_n"); 4509651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4510651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshrd_n_s64: { 4511651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1))); 4512651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAShr( 4513651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), 4514651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Amt->getZExtValue())), 45156bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines "shrd_n"); 4516651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4517651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshrd_n_u64: { 4518651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1))); 45196bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines uint64_t ShiftAmt = Amt->getZExtValue(); 45206bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // Right-shifting an unsigned value by its size yields 0. 45216bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (ShiftAmt == 64) 45226bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return ConstantInt::get(Int64Ty, 0); 45236bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt), 45246bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines "shrd_n"); 4525651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4526651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsrad_n_s64: { 4527651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2))); 4528651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateAShr( 4529651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), 4530651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Amt->getZExtValue())), 45316bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines "shrd_n"); 4532651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Ops[0], Ops[1]); 4533651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4534651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsrad_n_u64: { 4535651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2))); 45366bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines uint64_t ShiftAmt = Amt->getZExtValue(); 45376bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // Right-shifting an unsigned value by its size yields 0. 45386bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // As Op + 0 = Op, return Ops[0] directly. 45396bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (ShiftAmt == 64) 45406bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return Ops[0]; 45416bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt), 45426bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines "shrd_n"); 4543651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Ops[0], Ops[1]); 4544651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4545651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlalh_lane_s16: 4546651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlalh_laneq_s16: 4547651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlslh_lane_s16: 4548651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: { 4549651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), 4550651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "lane"); 4551651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value *, 2> ProductOps; 4552651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps.push_back(vectorWrapScalar16(Ops[1])); 4553651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps.push_back(vectorWrapScalar16(Ops[2])); 4554651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4); 45556bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), 4556651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps, "vqdmlXl"); 4557c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Constant *CI = ConstantInt::get(SizeTy, 0); 4558651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); 4559651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.pop_back(); 4560651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4561651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 || 4562651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16) 45636bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ? Intrinsic::aarch64_neon_sqadd 45646bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines : Intrinsic::aarch64_neon_sqsub; 4565651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops, "vqdmlXl"); 4566651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4567651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlals_s32: 4568651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlsls_s32: { 4569651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value *, 2> ProductOps; 4570651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps.push_back(Ops[1]); 4571651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps.push_back(EmitScalarExpr(E->getArg(2))); 4572651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = 45736bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar), 4574651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps, "vqdmlXl"); 4575651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4576651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32 45776bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ? Intrinsic::aarch64_neon_sqadd 45786bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines : Intrinsic::aarch64_neon_sqsub; 4579651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops, "vqdmlXl"); 4580651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4581651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlals_lane_s32: 4582651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlals_laneq_s32: 4583651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlsls_lane_s32: 4584651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: { 4585651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), 4586651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "lane"); 4587651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value *, 2> ProductOps; 4588651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps.push_back(Ops[1]); 4589651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps.push_back(Ops[2]); 4590651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = 45916bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar), 4592651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps, "vqdmlXl"); 4593651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.pop_back(); 4594651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4595651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 || 4596651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32) 45976bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ? Intrinsic::aarch64_neon_sqadd 45986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines : Intrinsic::aarch64_neon_sqsub; 4599651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl"); 4600651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 460199c40bb13b523d58d7aeb6446e4f486d6918ca58Nate Begeman } 4602e140af3e27016f902146023fba7680b43043ec07Rafael Espindola 46038b418685e9e4f02f4eb2a76e1ec063e07552b68dChris Lattner llvm::VectorType *VTy = GetNeonType(this, Type); 46049cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *Ty = VTy; 4605e140af3e27016f902146023fba7680b43043ec07Rafael Espindola if (!Ty) 46066bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 4607e140af3e27016f902146023fba7680b43043ec07Rafael Espindola 46086bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // Not all intrinsics handled by the common case work for AArch64 yet, so only 4609651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // defer to common code if it's been added to our special map. 46106bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Builtin = findNeonIntrinsicInMap(AArch64SIMDIntrinsicMap, BuiltinID, 46116bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines AArch64SIMDIntrinsicsProvenSorted); 4612651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4613651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Builtin) 4614651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitCommonNeonBuiltinExpr( 4615651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic, 46166bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Builtin->NameHint, Builtin->TypeModifier, E, Ops, nullptr); 4617651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 46186bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops)) 4619651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return V; 4620651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4621e140af3e27016f902146023fba7680b43043ec07Rafael Espindola unsigned Int; 4622e140af3e27016f902146023fba7680b43043ec07Rafael Espindola switch (BuiltinID) { 46236bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines default: return nullptr; 4624651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vbsl_v: 4625651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vbslq_v: { 4626651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *BitTy = llvm::VectorType::getInteger(VTy); 4627651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl"); 4628651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl"); 4629651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl"); 4630651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4631651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl"); 4632651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl"); 4633651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl"); 4634651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateBitCast(Ops[0], Ty); 4635651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4636651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfma_lane_v: 4637651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfmaq_lane_v: { // Only used for FP types 4638651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The ARM builtins (and instructions) have the addend as the first 4639651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // operand, but the 'fma' intrinsics have it last. Swap it around here. 4640651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Addend = Ops[0]; 4641651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Multiplicand = Ops[1]; 4642651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *LaneSource = Ops[2]; 4643651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Multiplicand; 4644651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = LaneSource; 4645651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Addend; 4646651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4647651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Now adjust things to handle the lane access. 4648651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v ? 4649651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(VTy->getElementType(), VTy->getNumElements() / 2) : 4650651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy; 4651651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Constant *cst = cast<Constant>(Ops[3]); 4652651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *SV = llvm::ConstantVector::getSplat(VTy->getNumElements(), cst); 4653651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy); 4654651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane"); 4655651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4656651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.pop_back(); 4657651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Intrinsic::fma; 4658651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmla"); 4659651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4660651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfma_laneq_v: { 4661651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType *VTy = cast<llvm::VectorType>(Ty); 4662651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // v1f64 fma should be mapped to Neon scalar f64 fma 4663651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (VTy && VTy->getElementType() == DoubleTy) { 4664651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 4665651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); 4666651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *VTy = GetNeonType(this, 4667651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags(NeonTypeFlags::Float64, false, true)); 4668651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateBitCast(Ops[2], VTy); 4669651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); 4670651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *F = CGM.getIntrinsic(Intrinsic::fma, DoubleTy); 4671651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Result = Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]); 4672651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateBitCast(Result, Ty); 4673651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4674651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 4675651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4676651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 46776bf1e8eadd36f077cf5403d23c664129d98eaf79Tim Northover 4678651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *STy = llvm::VectorType::get(VTy->getElementType(), 4679651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy->getNumElements() * 2); 4680651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateBitCast(Ops[2], STy); 4681651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value* SV = llvm::ConstantVector::getSplat(VTy->getNumElements(), 4682651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines cast<ConstantInt>(Ops[3])); 4683651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane"); 46846bf1e8eadd36f077cf5403d23c664129d98eaf79Tim Northover 4685651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall3(F, Ops[2], Ops[1], Ops[0]); 4686651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4687651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfmaq_laneq_v: { 4688651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 4689651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4690651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 46916bf1e8eadd36f077cf5403d23c664129d98eaf79Tim Northover 4692651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 4693651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3])); 4694651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall3(F, Ops[2], Ops[1], Ops[0]); 46956bf1e8eadd36f077cf5403d23c664129d98eaf79Tim Northover } 4696651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfmas_lane_f32: 4697651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfmas_laneq_f32: 4698651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfmad_lane_f64: 4699651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfmad_laneq_f64: { 4700651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(3))); 4701651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty = ConvertType(E->getCallReturnType()); 4702651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 4703651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); 4704651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]); 470530d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman } 4706651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfms_v: 4707651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfmsq_v: { // Only used for FP types 4708651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: probably remove when we no longer support aarch64_simd.h 4709651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // (arm_neon.h delegates to vfma). 4710651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4711651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The ARM builtins (and instructions) have the addend as the first 4712651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // operand, but the 'fma' intrinsics have it last. Swap it around here. 4713651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Subtrahend = Ops[0]; 4714651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Multiplicand = Ops[2]; 4715651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Multiplicand; 4716651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Subtrahend; 4717651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], VTy); 4718651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateFNeg(Ops[1]); 4719651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Intrinsic::fma; 4720651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmls"); 472130d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman } 4722651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmull_v: 4723651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 47246bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull; 47256bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (Type.isPoly()) Int = Intrinsic::aarch64_neon_pmull; 4726651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); 4727651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmax_v: 4728651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxq_v: 4729651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 47306bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax; 47316bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax; 4732651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax"); 4733651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmin_v: 4734651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminq_v: 4735651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 47366bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin; 47376bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin; 4738651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin"); 4739651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vabd_v: 4740651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vabdq_v: 4741651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 47426bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd; 47436bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd; 4744651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd"); 4745651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpadal_v: 4746651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpadalq_v: { 4747651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned ArgElts = VTy->getNumElements(); 4748651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType()); 4749651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned BitWidth = EltTy->getBitWidth(); 4750651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *ArgTy = llvm::VectorType::get( 4751651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), BitWidth/2), 2*ArgElts); 4752651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type* Tys[2] = { VTy, ArgTy }; 47536bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp; 4754651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<llvm::Value*, 1> TmpOps; 4755651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TmpOps.push_back(Ops[1]); 4756651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(Int, Tys); 4757651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vpadal"); 4758651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType()); 4759651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(tmp, addend); 4760651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4761651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpmin_v: 4762651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpminq_v: 4763651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 47646bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp; 47656bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp; 4766651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin"); 4767651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpmax_v: 4768651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpmaxq_v: 4769651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 47706bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp; 47716bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp; 4772651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax"); 4773651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminnm_v: 4774651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminnmq_v: 47756bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_fminnm; 4776651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vminnm"); 4777651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxnm_v: 4778651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxnmq_v: 47796bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_fmaxnm; 4780651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm"); 4781651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrecpss_f32: { 4782651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *f32Type = llvm::Type::getFloatTy(getLLVMContext()); 4783651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 47846bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, f32Type), 4785651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vrecps"); 478630d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman } 4787651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrecpsd_f64: { 4788651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *f64Type = llvm::Type::getDoubleTy(getLLVMContext()); 4789651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 47906bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, f64Type), 4791651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vrecps"); 4792651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4793651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshrun_n_v: 47946bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_sqshrun; 4795651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrun_n"); 4796651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqrshrun_n_v: 47976bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_sqrshrun; 4798651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrun_n"); 4799651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshrn_n_v: 48006bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn; 4801651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n"); 4802651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrshrn_n_v: 48036bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_rshrn; 4804651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshrn_n"); 4805651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqrshrn_n_v: 48066bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn; 4807651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n"); 4808651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrnda_v: 4809651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndaq_v: { 4810651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Intrinsic::round; 4811651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnda"); 4812651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4813651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndi_v: 4814651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndiq_v: { 4815651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Intrinsic::nearbyint; 4816651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndi"); 4817651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4818651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndm_v: 4819651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndmq_v: { 4820651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Intrinsic::floor; 4821651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndm"); 4822651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4823651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndn_v: 4824651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndnq_v: { 48256bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_frintn; 4826651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn"); 4827651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4828651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndp_v: 4829651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndpq_v: { 4830651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Intrinsic::ceil; 4831651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndp"); 4832651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4833651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndx_v: 4834651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndxq_v: { 4835651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Intrinsic::rint; 4836651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndx"); 4837651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4838651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrnd_v: 4839651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndq_v: { 4840651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Intrinsic::trunc; 4841651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndz"); 4842651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4843651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqz_v: 4844651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqzq_v: 4845651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OEQ, 4846651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ICmpInst::ICMP_EQ, "vceqz"); 4847651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgez_v: 4848651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgezq_v: 4849651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGE, 4850651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ICmpInst::ICMP_SGE, "vcgez"); 4851651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclez_v: 4852651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclezq_v: 4853651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLE, 4854651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ICmpInst::ICMP_SLE, "vclez"); 4855651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtz_v: 4856651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtzq_v: 4857651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT, 4858651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ICmpInst::ICMP_SGT, "vcgtz"); 4859651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltz_v: 4860651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltzq_v: 4861651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT, 4862651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ICmpInst::ICMP_SLT, "vcltz"); 4863651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_f64_v: 4864651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_f64_v: 486530d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4866651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float64, false, quad)); 4867258f930227c1a102c9c22eee88df65f748863425Jim Grosbach return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") 48689eb65a56e18bee1e5392bf2dff01cbd7b895f685Nate Begeman : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); 4869651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_f64_f32: { 4870651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines assert(Type.getEltType() == NeonTypeFlags::Float64 && quad && 4871651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "unexpected vcvt_f64_f32 builtin"); 4872651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float32, false, false); 4873651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); 4874651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4875651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFPExt(Ops[0], Ty, "vcvt"); 48769eb65a56e18bee1e5392bf2dff01cbd7b895f685Nate Begeman } 4877651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_f32_f64: { 4878651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines assert(Type.getEltType() == NeonTypeFlags::Float32 && 4879651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "unexpected vcvt_f32_f64 builtin"); 4880651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float64, false, true); 4881651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); 4882651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4883651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt"); 48849eb65a56e18bee1e5392bf2dff01cbd7b895f685Nate Begeman } 4885651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_s32_v: 4886651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_u32_v: 4887651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_s64_v: 4888651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_u64_v: 4889651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_s32_v: 4890651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_u32_v: 4891651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_s64_v: 4892651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_u64_v: { 4893651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Double = 4894651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 4895651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *InTy = 4896651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines GetNeonType(this, 4897651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags(Double ? NeonTypeFlags::Float64 4898651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : NeonTypeFlags::Float32, false, quad)); 4899651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], InTy); 4900651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (usgn) 4901651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFPToUI(Ops[0], Ty); 4902651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFPToSI(Ops[0], Ty); 4903651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4904651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvta_s32_v: 4905651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtaq_s32_v: 4906651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvta_u32_v: 4907651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtaq_u32_v: 4908651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvta_s64_v: 4909651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtaq_s64_v: 4910651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvta_u64_v: 4911651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtaq_u64_v: { 49126bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas; 4913651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Double = 4914651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 4915651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *InTy = 4916651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines GetNeonType(this, 4917651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags(Double ? NeonTypeFlags::Float64 4918651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : NeonTypeFlags::Float32, false, quad)); 4919651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, InTy }; 4920651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta"); 4921651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4922651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtm_s32_v: 4923651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtmq_s32_v: 4924651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtm_u32_v: 4925651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtmq_u32_v: 4926651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtm_s64_v: 4927651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtmq_s64_v: 4928651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtm_u64_v: 4929651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtmq_u64_v: { 49306bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms; 4931651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Double = 4932651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 4933651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *InTy = 4934651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines GetNeonType(this, 4935651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags(Double ? NeonTypeFlags::Float64 4936651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : NeonTypeFlags::Float32, false, quad)); 4937651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, InTy }; 4938651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm"); 4939651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4940651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtn_s32_v: 4941651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtnq_s32_v: 4942651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtn_u32_v: 4943651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtnq_u32_v: 4944651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtn_s64_v: 4945651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtnq_s64_v: 4946651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtn_u64_v: 4947651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtnq_u64_v: { 49486bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns; 4949651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Double = 4950651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 4951651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *InTy = 4952651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines GetNeonType(this, 4953651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags(Double ? NeonTypeFlags::Float64 4954651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : NeonTypeFlags::Float32, false, quad)); 4955651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, InTy }; 4956651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn"); 4957651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4958651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtp_s32_v: 4959651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtpq_s32_v: 4960651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtp_u32_v: 4961651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtpq_u32_v: 4962651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtp_s64_v: 4963651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtpq_s64_v: 4964651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtp_u64_v: 4965651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtpq_u64_v: { 49666bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps; 4967651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Double = 4968651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); 4969651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *InTy = 4970651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines GetNeonType(this, 4971651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags(Double ? NeonTypeFlags::Float64 4972651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : NeonTypeFlags::Float32, false, quad)); 4973651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, InTy }; 4974651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp"); 4975651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4976651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmulx_v: 4977651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmulxq_v: { 49786bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_fmulx; 4979651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmulx"); 498030d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman } 4981651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmul_lane_v: 4982651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmul_laneq_v: { 4983651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // v1f64 vmul_lane should be mapped to Neon scalar mul lane 4984651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Quad = false; 4985651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v) 4986651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Quad = true; 4987651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 4988651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *VTy = GetNeonType(this, 4989651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags(NeonTypeFlags::Float64, false, Quad)); 4990651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], VTy); 4991651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract"); 4992651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Result = Builder.CreateFMul(Ops[0], Ops[1]); 4993651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateBitCast(Result, Ty); 4994651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4995651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vnegd_s64: 4996651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateNeg(EmitScalarExpr(E->getArg(0)), "vnegd"); 4997651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpmaxnm_v: 4998651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpmaxnmq_v: { 49996bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_fmaxnmp; 5000651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmaxnm"); 5001651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5002651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpminnm_v: 5003651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpminnmq_v: { 50046bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_fminnmp; 5005651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpminnm"); 5006651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5007651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsqrt_v: 5008651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsqrtq_v: { 5009651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Intrinsic::sqrt; 501030d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5011651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqrt"); 50121c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman } 5013651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrbit_v: 5014651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrbitq_v: { 50156bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_rbit; 5016651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrbit"); 5017651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5018651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddv_u8: 5019651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: These are handled by the AArch64 scalar code. 5020651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines usgn = true; 5021651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FALLTHROUGH 5022651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddv_s8: { 50236bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 5024651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5025651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5026651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); 5027651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5028651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5029651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 5030651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5031651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 8)); 5032651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5033651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddv_u16: 5034651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines usgn = true; 5035651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FALLTHROUGH 5036651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddv_s16: { 50376bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 5038651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5039651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5040651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); 5041651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5042651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5043651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 5044651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5045651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 5046651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5047651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddvq_u8: 5048651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines usgn = true; 5049651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FALLTHROUGH 5050651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddvq_s8: { 50516bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 5052651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5053651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5054651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); 5055651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5056651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5057651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 5058651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5059651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 8)); 5060651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5061651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddvq_u16: 5062651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines usgn = true; 5063651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FALLTHROUGH 5064651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddvq_s16: { 50656bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 5066651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5067651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5068651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); 5069651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5070651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5071651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 5072651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5073651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 5074651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5075651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxv_u8: { 50766bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_umaxv; 5077651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5078651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5079651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); 5080651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5081651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5082651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5083651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5084651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 8)); 5085651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5086651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxv_u16: { 50876bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_umaxv; 5088651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5089651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5090651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); 5091651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5092651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5093651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5094651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5095651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 5096651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5097651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxvq_u8: { 50986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_umaxv; 5099651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5100651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5101651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); 5102651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5103651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5104651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5105651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5106651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 8)); 5107651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5108651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxvq_u16: { 51096bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_umaxv; 5110651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5111651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5112651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); 5113651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5114651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5115651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5116651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5117651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 5118651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5119651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxv_s8: { 51206bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_smaxv; 5121651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5122651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5123651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); 5124651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5125651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5126651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5127651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5128651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 8)); 5129651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5130651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxv_s16: { 51316bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_smaxv; 5132651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5133651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5134651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); 5135651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5136651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5137651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5138651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5139651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 5140651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5141651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxvq_s8: { 51426bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_smaxv; 5143651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5144651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5145651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); 5146651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5147651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5148651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5149651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5150651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 8)); 5151651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5152651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxvq_s16: { 51536bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_smaxv; 5154651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5155651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5156651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); 5157651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5158651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5159651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5160651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5161651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 5162651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5163651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminv_u8: { 51646bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_uminv; 5165651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5166651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5167651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); 5168651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5169651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5170651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5171651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5172651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 8)); 5173651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5174651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminv_u16: { 51756bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_uminv; 5176651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5177651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5178651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); 5179651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5180651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5181651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5182651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5183651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 5184651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5185651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminvq_u8: { 51866bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_uminv; 5187651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5188651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5189651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); 5190651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5191651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5192651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5193651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5194651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 8)); 5195651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5196651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminvq_u16: { 51976bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_uminv; 5198651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5199651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5200651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); 5201651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5202651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5203651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5204651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5205651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 5206651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5207651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminv_s8: { 52086bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_sminv; 5209651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5210651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5211651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); 5212651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5213651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5214651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5215651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5216651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 8)); 5217651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5218651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminv_s16: { 52196bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_sminv; 5220651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5221651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5222651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); 5223651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5224651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5225651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5226651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5227651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 5228651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5229651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminvq_s8: { 52306bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_sminv; 5231651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5232651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5233651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); 5234651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5235651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5236651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5237651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5238651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 8)); 5239651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5240651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminvq_s16: { 52416bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_sminv; 5242651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5243651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5244651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); 5245651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5246651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5247651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5248651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5249651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 5250651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5251651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmul_n_f64: { 5252651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 5253651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *RHS = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), DoubleTy); 5254651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFMul(Ops[0], RHS); 5255651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5256651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddlv_u8: { 52576bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_uaddlv; 5258651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5259651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5260651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); 5261651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5262651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5263651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5264651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5265651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 5266651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5267651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddlv_u16: { 52686bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_uaddlv; 5269651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5270651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5271651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); 5272651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5273651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5274651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5275651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5276651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddlvq_u8: { 52776bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_uaddlv; 5278651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5279651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5280651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); 5281651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5282651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5283651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5284651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5285651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 5286651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5287651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddlvq_u16: { 52886bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_uaddlv; 5289651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5290651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5291651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); 5292651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5293651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5294651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5295651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5296651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddlv_s8: { 52976bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_saddlv; 5298651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5299651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5300651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); 5301651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5302651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5303651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5304651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5305651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 5306651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5307651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddlv_s16: { 53086bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_saddlv; 5309651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5310651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5311651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); 5312651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5313651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5314651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5315651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5316651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddlvq_s8: { 53176bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_saddlv; 5318651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5319651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5320651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); 5321651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5322651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5323651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5324651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], 5325651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 16)); 5326651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5327651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddlvq_s16: { 53286bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_saddlv; 5329651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::IntegerType::get(getLLVMContext(), 32); 5330651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy = 5331651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); 5332651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5333651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5334651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5335651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5336651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsri_n_v: 5337651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsriq_n_v: { 53386bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_vsri; 5339651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty); 5340651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(Intrin, Ops, "vsri_n"); 5341651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5342651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsli_n_v: 5343651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsliq_n_v: { 53446bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_vsli; 5345651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty); 5346651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(Intrin, Ops, "vsli_n"); 5347651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5348651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsra_n_v: 5349651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsraq_n_v: 5350651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5351651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); 5352651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Ops[0], Ops[1]); 5353651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrsra_n_v: 5354651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrsraq_n_v: { 53556bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl; 5356651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<llvm::Value*,2> TmpOps; 5357651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TmpOps.push_back(Ops[1]); 5358651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TmpOps.push_back(Ops[2]); 5359651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function* F = CGM.getIntrinsic(Int, Ty); 5360651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vrshr_n", 1, true); 5361651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], VTy); 5362651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Ops[0], tmp); 5363651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5364651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: Sharing loads & stores with 32-bit is complicated by the absence 5365651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // of an Align parameter here. 5366651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_x2_v: 5367651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_x2_v: 5368651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_x3_v: 5369651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_x3_v: 5370651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_x4_v: 5371651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_x4_v: { 5372651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType()); 5373651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5374651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, PTy }; 5375651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Int; 5376651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 5377651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_x2_v: 5378651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_x2_v: 53796bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_ld1x2; 5380651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 5381651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_x3_v: 5382651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_x3_v: 53836bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_ld1x3; 5384651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 5385651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_x4_v: 5386651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_x4_v: 53876bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_ld1x4; 5388651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 5389550a9d823a939366a9f776b58f18883acd905a93Bob Wilson } 5390651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(Int, Tys); 5391651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN"); 5392651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 5393651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5394651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateStore(Ops[1], Ops[0]); 5395651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5396651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_x2_v: 5397651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_x2_v: 5398651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_x3_v: 5399651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_x3_v: 5400651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_x4_v: 5401651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_x4_v: { 5402651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType()); 5403651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, PTy }; 5404651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Int; 5405651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 5406651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_x2_v: 5407651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_x2_v: 54086bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_st1x2; 5409651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 5410651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_x3_v: 5411651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_x3_v: 54126bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_st1x3; 5413651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 5414651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_x4_v: 5415651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_x4_v: 54166bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_st1x4; 5417651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 5418651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5419651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value *, 4> IntOps(Ops.begin()+1, Ops.end()); 5420651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines IntOps.push_back(Ops[0]); 5421651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), IntOps, ""); 5422651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5423651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_v: 5424651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_v: 5425651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); 5426651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateLoad(Ops[0]); 5427651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_v: 5428651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_v: 5429651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); 5430651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], VTy); 5431651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateStore(Ops[1], Ops[0]); 5432651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_lane_v: 5433651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_lane_v: 54344be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 54354be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ty = llvm::PointerType::getUnqual(VTy->getElementType()); 54364be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5437651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateLoad(Ops[0]); 5438651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane"); 5439651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_dup_v: 5440651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_dup_v: { 54414be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Value *V = UndefValue::get(Ty); 54424be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ty = llvm::PointerType::getUnqual(VTy->getElementType()); 54434be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5444651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateLoad(Ops[0]); 544577b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner llvm::Constant *CI = ConstantInt::get(Int32Ty, 0); 5446651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI); 54474be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman return EmitNeonSplat(Ops[0], CI); 54484be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 5449651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_lane_v: 5450651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_lane_v: 5451651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 5452651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); 54534be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 5454651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateStore(Ops[1], Builder.CreateBitCast(Ops[0], Ty)); 5455651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_v: 5456651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2q_v: { 5457651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *PTy = llvm::PointerType::getUnqual(VTy); 5458651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5459651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, PTy }; 54606bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2, Tys); 5461651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); 5462651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 5463651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(Ops[1]->getType())); 54644be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman return Builder.CreateStore(Ops[1], Ops[0]); 54654be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 5466651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_v: 5467651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3q_v: { 5468651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *PTy = llvm::PointerType::getUnqual(VTy); 5469651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5470651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, PTy }; 54716bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3, Tys); 5472651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); 5473651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 5474651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(Ops[1]->getType())); 54754be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman return Builder.CreateStore(Ops[1], Ops[0]); 54764be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 5477651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_v: 5478651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4q_v: { 5479651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *PTy = llvm::PointerType::getUnqual(VTy); 5480651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5481651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, PTy }; 54826bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4, Tys); 5483651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); 5484651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 5485651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(Ops[1]->getType())); 5486651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateStore(Ops[1], Ops[0]); 5487651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5488651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_dup_v: 5489651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2q_dup_v: { 5490651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *PTy = 5491651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(VTy->getElementType()); 5492651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5493651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, PTy }; 54946bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2r, Tys); 5495651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); 5496651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 5497651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(Ops[1]->getType())); 54984be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman return Builder.CreateStore(Ops[1], Ops[0]); 54994be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 5500651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_dup_v: 5501651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3q_dup_v: { 5502651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *PTy = 5503651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(VTy->getElementType()); 5504651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5505651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, PTy }; 55066bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3r, Tys); 5507651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); 5508651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 5509651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(Ops[1]->getType())); 5510651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateStore(Ops[1], Ops[0]); 5511651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5512651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_dup_v: 5513651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4q_dup_v: { 5514651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *PTy = 5515651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(VTy->getElementType()); 5516651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5517651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, PTy }; 55186bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4r, Tys); 5519651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); 5520651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 5521651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(Ops[1]->getType())); 5522651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateStore(Ops[1], Ops[0]); 5523651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5524651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_lane_v: 5525651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2q_lane_v: { 5526651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; 55276bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2lane, Tys); 5528651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[1]); 5529651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()+1); 5530651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 55314be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 5532651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[3] = Builder.CreateZExt(Ops[3], 5533651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 64)); 5534176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld2_lane"); 55354be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 55364be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 55374be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman return Builder.CreateStore(Ops[1], Ops[0]); 55384be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 5539651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_lane_v: 5540651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3q_lane_v: { 5541651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; 55426bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3lane, Tys); 5543651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[1]); 5544651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()+1); 5545651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 55464be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 55474be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[3] = Builder.CreateBitCast(Ops[3], Ty); 5548651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[4] = Builder.CreateZExt(Ops[4], 5549651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 64)); 5550176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane"); 55514be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 55524be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 55534be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman return Builder.CreateStore(Ops[1], Ops[0]); 55544be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 5555651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_lane_v: 5556651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4q_lane_v: { 5557651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; 55586bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4lane, Tys); 5559651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[1]); 5560651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()+1); 5561651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 55624be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 55634be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[3] = Builder.CreateBitCast(Ops[3], Ty); 55644be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[4] = Builder.CreateBitCast(Ops[4], Ty); 5565651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[5] = Builder.CreateZExt(Ops[5], 5566651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 64)); 5567176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld4_lane"); 55684be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 55694be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 55704be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman return Builder.CreateStore(Ops[1], Ops[0]); 55714be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 5572651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2_v: 5573651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2q_v: { 5574651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[0]); 5575651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()); 5576651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[2]->getType() }; 55776bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2, Tys), 5578464ccb68f22a7e1c0a2844551c16f721540c91c3Nate Begeman Ops, ""); 5579eac1f6746a2915fea3ed42284b07e274c0095a95Bob Wilson } 5580651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2_lane_v: 5581651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2q_lane_v: { 5582651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[0]); 5583651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()); 5584651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateZExt(Ops[2], 5585651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 64)); 5586651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; 55876bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2lane, Tys), 5588464ccb68f22a7e1c0a2844551c16f721540c91c3Nate Begeman Ops, ""); 5589651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5590651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3_v: 5591651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3q_v: { 5592651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[0]); 5593651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()); 5594651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; 55956bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3, Tys), 5596464ccb68f22a7e1c0a2844551c16f721540c91c3Nate Begeman Ops, ""); 5597651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5598651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3_lane_v: 5599651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3q_lane_v: { 5600651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[0]); 5601651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()); 5602651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[3] = Builder.CreateZExt(Ops[3], 5603651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 64)); 5604651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; 56056bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3lane, Tys), 5606464ccb68f22a7e1c0a2844551c16f721540c91c3Nate Begeman Ops, ""); 5607651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5608651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4_v: 5609651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4q_v: { 5610651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[0]); 5611651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()); 5612651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; 56136bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4, Tys), 5614464ccb68f22a7e1c0a2844551c16f721540c91c3Nate Begeman Ops, ""); 56156bf1e8eadd36f077cf5403d23c664129d98eaf79Tim Northover } 5616651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4_lane_v: 5617651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4q_lane_v: { 5618651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[0]); 5619651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()); 5620651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[4] = Builder.CreateZExt(Ops[4], 5621651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), 64)); 5622651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[5]->getType() }; 56236bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4lane, Tys), 5624651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, ""); 56251c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman } 5626651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtrn_v: 5627651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtrnq_v: { 56284be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 56294be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 56304be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 56316bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *SV = nullptr; 56324be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman 56331c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman for (unsigned vi = 0; vi != 2; ++vi) { 56344be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman SmallVector<Constant*, 16> Indices; 56354be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 5636651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Indices.push_back(ConstantInt::get(Int32Ty, i+vi)); 5637651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Indices.push_back(ConstantInt::get(Int32Ty, i+e+vi)); 56381c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman } 56394be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); 5640fb018d153cbf12fd2a4a278cbf7614b9a2e2835eChris Lattner SV = llvm::ConstantVector::get(Indices); 56414be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn"); 56424be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman SV = Builder.CreateStore(SV, Addr); 56431c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman } 56444be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman return SV; 56451c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman } 5646651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vuzp_v: 5647651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vuzpq_v: { 56484be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 56491c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 56504be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 56516bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *SV = nullptr; 5652258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 56534be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman for (unsigned vi = 0; vi != 2; ++vi) { 56544be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman SmallVector<Constant*, 16> Indices; 56554be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) 565677b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner Indices.push_back(ConstantInt::get(Int32Ty, 2*i+vi)); 56574be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman 56584be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); 5659fb018d153cbf12fd2a4a278cbf7614b9a2e2835eChris Lattner SV = llvm::ConstantVector::get(Indices); 56604be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp"); 56614be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman SV = Builder.CreateStore(SV, Addr); 56624be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 56634be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman return SV; 56641c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman } 5665651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vzip_v: 5666651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vzipq_v: { 56674be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 56681c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 56694be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 56706bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *SV = nullptr; 5671258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 56724be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman for (unsigned vi = 0; vi != 2; ++vi) { 56734be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman SmallVector<Constant*, 16> Indices; 56744be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 5675e361cc3daa6c22e4413d48bd8b8934ea9fd5a55fDaniel Dunbar Indices.push_back(ConstantInt::get(Int32Ty, (i + vi*e) >> 1)); 5676e361cc3daa6c22e4413d48bd8b8934ea9fd5a55fDaniel Dunbar Indices.push_back(ConstantInt::get(Int32Ty, ((i + vi*e) >> 1)+e)); 56774be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 56784be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); 5679fb018d153cbf12fd2a4a278cbf7614b9a2e2835eChris Lattner SV = llvm::ConstantVector::get(Indices); 56804be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip"); 56814be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman SV = Builder.CreateStore(SV, Addr); 56824be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 56834be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman return SV; 56849eb65a56e18bee1e5392bf2dff01cbd7b895f685Nate Begeman } 5685651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl1q_v: { 56866bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl1, Ty), 5687651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbl1"); 5688651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5689651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl2q_v: { 56906bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl2, Ty), 5691651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbl2"); 5692651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5693651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl3q_v: { 56946bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl3, Ty), 5695651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbl3"); 5696651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5697651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl4q_v: { 56986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl4, Ty), 5699651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbl4"); 5700651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5701651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx1q_v: { 57026bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx1, Ty), 5703651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbx1"); 5704651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5705651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx2q_v: { 57066bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx2, Ty), 5707651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbx2"); 5708651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5709651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx3q_v: { 57106bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx3, Ty), 5711651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbx3"); 5712651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5713651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx4q_v: { 57146bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx4, Ty), 5715651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbx4"); 5716651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5717651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsqadd_v: 5718651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsqaddq_v: { 57196bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_usqadd; 5720651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqadd"); 5721651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5722651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vuqadd_v: 5723651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vuqaddq_v: { 57246bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_suqadd; 5725651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vuqadd"); 5726651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 57272752c0137d95aa2f4ee1cdff4b564bac842e041bChris Lattner } 57282752c0137d95aa2f4ee1cdff4b564bac842e041bChris Lattner} 57292752c0137d95aa2f4ee1cdff4b564bac842e041bChris Lattner 5730aa51e513850688b7963efc62abf1eface7037602Bill Wendlingllvm::Value *CodeGenFunction:: 5731795b10062c2eaffae9e04241fb1a73cdbcb24a37Bill WendlingBuildVector(ArrayRef<llvm::Value*> Ops) { 5732aa51e513850688b7963efc62abf1eface7037602Bill Wendling assert((Ops.size() & (Ops.size() - 1)) == 0 && 5733aa51e513850688b7963efc62abf1eface7037602Bill Wendling "Not a power-of-two sized vector!"); 5734aa51e513850688b7963efc62abf1eface7037602Bill Wendling bool AllConstants = true; 5735aa51e513850688b7963efc62abf1eface7037602Bill Wendling for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i) 5736aa51e513850688b7963efc62abf1eface7037602Bill Wendling AllConstants &= isa<Constant>(Ops[i]); 5737aa51e513850688b7963efc62abf1eface7037602Bill Wendling 5738aa51e513850688b7963efc62abf1eface7037602Bill Wendling // If this is a constant vector, create a ConstantVector. 5739aa51e513850688b7963efc62abf1eface7037602Bill Wendling if (AllConstants) { 57402ce8842641cc312628c4be836d34eb250e7c3f78Chris Lattner SmallVector<llvm::Constant*, 16> CstOps; 5741aa51e513850688b7963efc62abf1eface7037602Bill Wendling for (unsigned i = 0, e = Ops.size(); i != e; ++i) 5742aa51e513850688b7963efc62abf1eface7037602Bill Wendling CstOps.push_back(cast<Constant>(Ops[i])); 5743aa51e513850688b7963efc62abf1eface7037602Bill Wendling return llvm::ConstantVector::get(CstOps); 5744aa51e513850688b7963efc62abf1eface7037602Bill Wendling } 5745aa51e513850688b7963efc62abf1eface7037602Bill Wendling 5746aa51e513850688b7963efc62abf1eface7037602Bill Wendling // Otherwise, insertelement the values to build the vector. 5747aa51e513850688b7963efc62abf1eface7037602Bill Wendling Value *Result = 5748aa51e513850688b7963efc62abf1eface7037602Bill Wendling llvm::UndefValue::get(llvm::VectorType::get(Ops[0]->getType(), Ops.size())); 5749aa51e513850688b7963efc62abf1eface7037602Bill Wendling 5750aa51e513850688b7963efc62abf1eface7037602Bill Wendling for (unsigned i = 0, e = Ops.size(); i != e; ++i) 57512ce8842641cc312628c4be836d34eb250e7c3f78Chris Lattner Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt32(i)); 5752aa51e513850688b7963efc62abf1eface7037602Bill Wendling 5753aa51e513850688b7963efc62abf1eface7037602Bill Wendling return Result; 5754aa51e513850688b7963efc62abf1eface7037602Bill Wendling} 5755aa51e513850688b7963efc62abf1eface7037602Bill Wendling 57561eb4433ac451dc16f4133a88af2d002ac26c58efMike StumpValue *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, 57571feedd84221e8dbcc3faf3de27cc42b559db845dChris Lattner const CallExpr *E) { 57585f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner SmallVector<Value*, 4> Ops; 57592929cfa9b7df1d5b0571b54161783e4d791a0b77Anders Carlsson 576046c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner // Find out if any arguments are required to be integer constant expressions. 576146c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner unsigned ICEArguments = 0; 576246c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner ASTContext::GetBuiltinTypeError Error; 576346c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); 576446c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner assert(Error == ASTContext::GE_None && "Should not codegen an error"); 576546c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner 576646c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) { 576746c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner // If this is a normal argument, just emit it as a scalar. 576846c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner if ((ICEArguments & (1 << i)) == 0) { 576946c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner Ops.push_back(EmitScalarExpr(E->getArg(i))); 577046c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner continue; 577146c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner } 577246c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner 577346c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner // If this is required to be a constant, constant fold it so that we know 577446c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner // that the generated intrinsic gets a ConstantInt. 577546c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner llvm::APSInt Result; 577646c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext()); 577746c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst; 5778d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); 577946c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner } 57802929cfa9b7df1d5b0571b54161783e4d791a0b77Anders Carlsson 5781564f1de67d7ba43646b8740db86d6269e3dfbe0bAnders Carlsson switch (BuiltinID) { 57826bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines default: return nullptr; 5783651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case X86::BI_mm_prefetch: { 5784651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Address = EmitScalarExpr(E->getArg(0)); 5785651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *RW = ConstantInt::get(Int32Ty, 0); 5786651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Locality = EmitScalarExpr(E->getArg(1)); 5787651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Data = ConstantInt::get(Int32Ty, 1); 5788651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *F = CGM.getIntrinsic(Intrinsic::prefetch); 5789651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall4(F, Address, RW, Locality, Data); 5790651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5791aa51e513850688b7963efc62abf1eface7037602Bill Wendling case X86::BI__builtin_ia32_vec_init_v8qi: 5792aa51e513850688b7963efc62abf1eface7037602Bill Wendling case X86::BI__builtin_ia32_vec_init_v4hi: 5793aa51e513850688b7963efc62abf1eface7037602Bill Wendling case X86::BI__builtin_ia32_vec_init_v2si: 5794aa51e513850688b7963efc62abf1eface7037602Bill Wendling return Builder.CreateBitCast(BuildVector(Ops), 5795d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall llvm::Type::getX86_MMXTy(getLLVMContext())); 57961944ec188408aff1931c62c79a069e30f2549ec2Argyrios Kyrtzidis case X86::BI__builtin_ia32_vec_ext_v2si: 57971944ec188408aff1931c62c79a069e30f2549ec2Argyrios Kyrtzidis return Builder.CreateExtractElement(Ops[0], 57981944ec188408aff1931c62c79a069e30f2549ec2Argyrios Kyrtzidis llvm::ConstantInt::get(Ops[1]->getType(), 0)); 5799e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman case X86::BI__builtin_ia32_ldmxcsr: { 58003fc779191ce87cddf603e0c5bdbd225611201ff4Juergen Ributzka Value *Tmp = CreateMemTemp(E->getArg(0)->getType()); 5801e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman Builder.CreateStore(Ops[0], Tmp); 5802e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr), 58033fc779191ce87cddf603e0c5bdbd225611201ff4Juergen Ributzka Builder.CreateBitCast(Tmp, Int8PtrTy)); 5804e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman } 5805e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman case X86::BI__builtin_ia32_stmxcsr: { 5806fe0af454943374758309a1066e3304b3a56af04eJuergen Ributzka Value *Tmp = CreateMemTemp(E->getType()); 5807012614ecf78442368ec82ee30efb3bc047b413e6Ted Kremenek Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr), 58083fc779191ce87cddf603e0c5bdbd225611201ff4Juergen Ributzka Builder.CreateBitCast(Tmp, Int8PtrTy)); 5809e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman return Builder.CreateLoad(Tmp, "stmxcsr"); 5810e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman } 5811e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman case X86::BI__builtin_ia32_storehps: 5812e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman case X86::BI__builtin_ia32_storelps: { 581377b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner llvm::Type *PtrTy = llvm::PointerType::getUnqual(Int64Ty); 581477b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2); 58151eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 5816e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman // cast val v2i64 5817e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman Ops[1] = Builder.CreateBitCast(Ops[1], VecTy, "cast"); 58181eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 5819e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman // extract (0, 1) 5820e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman unsigned Index = BuiltinID == X86::BI__builtin_ia32_storelps ? 0 : 1; 5821c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Idx = llvm::ConstantInt::get(SizeTy, Index); 5822e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman Ops[1] = Builder.CreateExtractElement(Ops[1], Idx, "extract"); 5823e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman 5824e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman // cast pointer to i64 & store 5825e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], PtrTy); 5826e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman return Builder.CreateStore(Ops[1], Ops[0]); 5827e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman } 582828cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling case X86::BI__builtin_ia32_palignr: { 582928cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); 5830258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 583128cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling // If palignr is shifting the pair of input vectors less than 9 bytes, 583228cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling // emit a shuffle instruction. 583328cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling if (shiftVal <= 8) { 58345f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner SmallVector<llvm::Constant*, 8> Indices; 583528cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling for (unsigned i = 0; i != 8; ++i) 583628cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling Indices.push_back(llvm::ConstantInt::get(Int32Ty, shiftVal + i)); 5837258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 5838fb018d153cbf12fd2a4a278cbf7614b9a2e2835eChris Lattner Value* SV = llvm::ConstantVector::get(Indices); 583928cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr"); 584028cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling } 5841258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 584228cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling // If palignr is shifting the pair of input vectors more than 8 but less 584328cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling // than 16 bytes, emit a logical right shift of the destination. 584428cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling if (shiftVal < 16) { 584528cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling // MMX has these as 1 x i64 vectors for some odd optimization reasons. 58462acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 1); 5847258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 584828cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); 584928cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling Ops[1] = llvm::ConstantInt::get(VecTy, (shiftVal-8) * 8); 5850258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 585128cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling // create i32 constant 585228cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_mmx_psrl_q); 5853176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return Builder.CreateCall(F, makeArrayRef(Ops.data(), 2), "palignr"); 585428cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling } 5855258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 58565c22ad2ef6bf39da22d5190025e0ddfd4b568b2aEli Friedman // If palignr is shifting the pair of vectors more than 16 bytes, emit zero. 585728cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling return llvm::Constant::getNullValue(ConvertType(E->getType())); 585828cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling } 5859c3420ffb282c6ffc0192013bf8045b6c21eddeceNate Begeman case X86::BI__builtin_ia32_palignr128: { 5860ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); 5861258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 5862ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman // If palignr is shifting the pair of input vectors less than 17 bytes, 5863ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman // emit a shuffle instruction. 5864ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman if (shiftVal <= 16) { 58655f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner SmallVector<llvm::Constant*, 16> Indices; 5866ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman for (unsigned i = 0; i != 16; ++i) 586777b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner Indices.push_back(llvm::ConstantInt::get(Int32Ty, shiftVal + i)); 5868258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 5869fb018d153cbf12fd2a4a278cbf7614b9a2e2835eChris Lattner Value* SV = llvm::ConstantVector::get(Indices); 5870ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr"); 5871ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman } 5872258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 5873ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman // If palignr is shifting the pair of input vectors more than 16 but less 5874ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman // than 32 bytes, emit a logical right shift of the destination. 5875ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman if (shiftVal < 32) { 58762acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2); 5877258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 5878ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); 587977b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner Ops[1] = llvm::ConstantInt::get(Int32Ty, (shiftVal-16) * 8); 5880258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 5881ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman // create i32 constant 5882ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_sse2_psrl_dq); 5883176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return Builder.CreateCall(F, makeArrayRef(Ops.data(), 2), "palignr"); 5884ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman } 5885258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 5886ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman // If palignr is shifting the pair of vectors more than 32 bytes, emit zero. 5887ce5818a19a8f77d1540d0352649d6687eca4af6bNate Begeman return llvm::Constant::getNullValue(ConvertType(E->getType())); 588891b59274439f776cdd545bd7bf5849fdb1842160Eric Christopher } 58899c2ffd803af03f1728423d0d73ff87d988642633Craig Topper case X86::BI__builtin_ia32_palignr256: { 58909c2ffd803af03f1728423d0d73ff87d988642633Craig Topper unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); 58919c2ffd803af03f1728423d0d73ff87d988642633Craig Topper 58929c2ffd803af03f1728423d0d73ff87d988642633Craig Topper // If palignr is shifting the pair of input vectors less than 17 bytes, 58939c2ffd803af03f1728423d0d73ff87d988642633Craig Topper // emit a shuffle instruction. 58949c2ffd803af03f1728423d0d73ff87d988642633Craig Topper if (shiftVal <= 16) { 58959c2ffd803af03f1728423d0d73ff87d988642633Craig Topper SmallVector<llvm::Constant*, 32> Indices; 58969c2ffd803af03f1728423d0d73ff87d988642633Craig Topper // 256-bit palignr operates on 128-bit lanes so we need to handle that 58979c2ffd803af03f1728423d0d73ff87d988642633Craig Topper for (unsigned l = 0; l != 2; ++l) { 58989c2ffd803af03f1728423d0d73ff87d988642633Craig Topper unsigned LaneStart = l * 16; 58999c2ffd803af03f1728423d0d73ff87d988642633Craig Topper unsigned LaneEnd = (l+1) * 16; 59009c2ffd803af03f1728423d0d73ff87d988642633Craig Topper for (unsigned i = 0; i != 16; ++i) { 59019c2ffd803af03f1728423d0d73ff87d988642633Craig Topper unsigned Idx = shiftVal + i + LaneStart; 59029c2ffd803af03f1728423d0d73ff87d988642633Craig Topper if (Idx >= LaneEnd) Idx += 16; // end of lane, switch operand 59039c2ffd803af03f1728423d0d73ff87d988642633Craig Topper Indices.push_back(llvm::ConstantInt::get(Int32Ty, Idx)); 59049c2ffd803af03f1728423d0d73ff87d988642633Craig Topper } 59059c2ffd803af03f1728423d0d73ff87d988642633Craig Topper } 59069c2ffd803af03f1728423d0d73ff87d988642633Craig Topper 59079c2ffd803af03f1728423d0d73ff87d988642633Craig Topper Value* SV = llvm::ConstantVector::get(Indices); 59089c2ffd803af03f1728423d0d73ff87d988642633Craig Topper return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr"); 59099c2ffd803af03f1728423d0d73ff87d988642633Craig Topper } 59109c2ffd803af03f1728423d0d73ff87d988642633Craig Topper 59119c2ffd803af03f1728423d0d73ff87d988642633Craig Topper // If palignr is shifting the pair of input vectors more than 16 but less 59129c2ffd803af03f1728423d0d73ff87d988642633Craig Topper // than 32 bytes, emit a logical right shift of the destination. 59139c2ffd803af03f1728423d0d73ff87d988642633Craig Topper if (shiftVal < 32) { 59149c2ffd803af03f1728423d0d73ff87d988642633Craig Topper llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 4); 59159c2ffd803af03f1728423d0d73ff87d988642633Craig Topper 59169c2ffd803af03f1728423d0d73ff87d988642633Craig Topper Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); 59179c2ffd803af03f1728423d0d73ff87d988642633Craig Topper Ops[1] = llvm::ConstantInt::get(Int32Ty, (shiftVal-16) * 8); 59189c2ffd803af03f1728423d0d73ff87d988642633Craig Topper 59199c2ffd803af03f1728423d0d73ff87d988642633Craig Topper // create i32 constant 59209c2ffd803af03f1728423d0d73ff87d988642633Craig Topper llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_avx2_psrl_dq); 5921176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return Builder.CreateCall(F, makeArrayRef(Ops.data(), 2), "palignr"); 59229c2ffd803af03f1728423d0d73ff87d988642633Craig Topper } 59239c2ffd803af03f1728423d0d73ff87d988642633Craig Topper 59249c2ffd803af03f1728423d0d73ff87d988642633Craig Topper // If palignr is shifting the pair of vectors more than 32 bytes, emit zero. 59259c2ffd803af03f1728423d0d73ff87d988642633Craig Topper return llvm::Constant::getNullValue(ConvertType(E->getType())); 59269c2ffd803af03f1728423d0d73ff87d988642633Craig Topper } 5927b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling case X86::BI__builtin_ia32_movntps: 59284a7376d00df98cbfcad29ee709637707a2f3d46cCraig Topper case X86::BI__builtin_ia32_movntps256: 5929b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling case X86::BI__builtin_ia32_movntpd: 59304a7376d00df98cbfcad29ee709637707a2f3d46cCraig Topper case X86::BI__builtin_ia32_movntpd256: 5931b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling case X86::BI__builtin_ia32_movntdq: 59324a7376d00df98cbfcad29ee709637707a2f3d46cCraig Topper case X86::BI__builtin_ia32_movntdq256: 5933440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman case X86::BI__builtin_ia32_movnti: 5934440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman case X86::BI__builtin_ia32_movnti64: { 5935b107dd02f5e365f60b036b8a27c784f2d0d0a855Bill Wendling llvm::MDNode *Node = llvm::MDNode::get(getLLVMContext(), 5936b107dd02f5e365f60b036b8a27c784f2d0d0a855Bill Wendling Builder.getInt32(1)); 5937b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling 5938b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling // Convert the type of the pointer to a pointer to the stored type. 5939b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling Value *BC = Builder.CreateBitCast(Ops[0], 5940b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling llvm::PointerType::getUnqual(Ops[1]->getType()), 5941b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling "cast"); 5942b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling StoreInst *SI = Builder.CreateStore(Ops[1], BC); 5943b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling SI->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node); 5944440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman 5945440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman // If the operand is an integer, we can't assume alignment. Otherwise, 5946440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman // assume natural alignment. 5947440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman QualType ArgTy = E->getArg(1)->getType(); 5948440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman unsigned Align; 5949440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman if (ArgTy->isIntegerType()) 5950440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman Align = 1; 5951440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman else 5952440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman Align = getContext().getTypeSizeInChars(ArgTy).getQuantity(); 5953440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman SI->setAlignment(Align); 5954b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling return SI; 5955b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling } 59568b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer // 3DNow! 59578b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer case X86::BI__builtin_ia32_pswapdsf: 59588b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer case X86::BI__builtin_ia32_pswapdsi: { 59596bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines const char *name = nullptr; 59608b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer Intrinsic::ID ID = Intrinsic::not_intrinsic; 59618b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer switch(BuiltinID) { 5962f8495d67ca4dd2ea15a4dc59e9a2fa32a9bfa475Craig Topper default: llvm_unreachable("Unsupported intrinsic!"); 59638b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer case X86::BI__builtin_ia32_pswapdsf: 59648b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer case X86::BI__builtin_ia32_pswapdsi: 59658b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer name = "pswapd"; 59668b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer ID = Intrinsic::x86_3dnowa_pswapd; 59678b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer break; 59688b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer } 5969345032a7211a6f983d59c30c0b3fa2b96819532aChandler Carruth llvm::Type *MMXTy = llvm::Type::getX86_MMXTy(getLLVMContext()); 5970345032a7211a6f983d59c30c0b3fa2b96819532aChandler Carruth Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy, "cast"); 59718b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer llvm::Function *F = CGM.getIntrinsic(ID); 59724c7d9f1507d0f102bd4133bba63348636facd469Jay Foad return Builder.CreateCall(F, Ops, name); 59738b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer } 59749a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer case X86::BI__builtin_ia32_rdrand16_step: 59759a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer case X86::BI__builtin_ia32_rdrand32_step: 59761bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao case X86::BI__builtin_ia32_rdrand64_step: 59771bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao case X86::BI__builtin_ia32_rdseed16_step: 59781bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao case X86::BI__builtin_ia32_rdseed32_step: 59791bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao case X86::BI__builtin_ia32_rdseed64_step: { 59809a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer Intrinsic::ID ID; 59819a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer switch (BuiltinID) { 59829a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer default: llvm_unreachable("Unsupported intrinsic!"); 59839a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer case X86::BI__builtin_ia32_rdrand16_step: 59849a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer ID = Intrinsic::x86_rdrand_16; 59859a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer break; 59869a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer case X86::BI__builtin_ia32_rdrand32_step: 59879a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer ID = Intrinsic::x86_rdrand_32; 59889a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer break; 59899a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer case X86::BI__builtin_ia32_rdrand64_step: 59909a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer ID = Intrinsic::x86_rdrand_64; 59919a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer break; 59921bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao case X86::BI__builtin_ia32_rdseed16_step: 59931bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao ID = Intrinsic::x86_rdseed_16; 59941bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao break; 59951bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao case X86::BI__builtin_ia32_rdseed32_step: 59961bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao ID = Intrinsic::x86_rdseed_32; 59971bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao break; 59981bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao case X86::BI__builtin_ia32_rdseed64_step: 59991bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao ID = Intrinsic::x86_rdseed_64; 60001bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao break; 60019a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer } 60029a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer 60039a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID)); 60049a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer Builder.CreateStore(Builder.CreateExtractValue(Call, 0), Ops[0]); 60059a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer return Builder.CreateExtractValue(Call, 1); 60069a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer } 60072766deb114cc5d3420027764438cf683dda8a9f0Juergen Ributzka // AVX2 broadcast 60082766deb114cc5d3420027764438cf683dda8a9f0Juergen Ributzka case X86::BI__builtin_ia32_vbroadcastsi256: { 60093fc779191ce87cddf603e0c5bdbd225611201ff4Juergen Ributzka Value *VecTmp = CreateMemTemp(E->getArg(0)->getType()); 60103fc779191ce87cddf603e0c5bdbd225611201ff4Juergen Ributzka Builder.CreateStore(Ops[0], VecTmp); 60113fc779191ce87cddf603e0c5bdbd225611201ff4Juergen Ributzka Value *F = CGM.getIntrinsic(Intrinsic::x86_avx2_vbroadcasti128); 60123fc779191ce87cddf603e0c5bdbd225611201ff4Juergen Ributzka return Builder.CreateCall(F, Builder.CreateBitCast(VecTmp, Int8PtrTy)); 60132766deb114cc5d3420027764438cf683dda8a9f0Juergen Ributzka } 6014564f1de67d7ba43646b8740db86d6269e3dfbe0bAnders Carlsson } 6015564f1de67d7ba43646b8740db86d6269e3dfbe0bAnders Carlsson} 6016564f1de67d7ba43646b8740db86d6269e3dfbe0bAnders Carlsson 60179631939f82c0eaa6fb3936a0ce58a41adfbc9011Tony Linthicum 60181eb4433ac451dc16f4133a88af2d002ac26c58efMike StumpValue *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, 60191feedd84221e8dbcc3faf3de27cc42b559db845dChris Lattner const CallExpr *E) { 60205f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner SmallVector<Value*, 4> Ops; 6021dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner 6022dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) 6023dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner Ops.push_back(EmitScalarExpr(E->getArg(i))); 6024dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner 6025dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner Intrinsic::ID ID = Intrinsic::not_intrinsic; 6026dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner 6027dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner switch (BuiltinID) { 60286bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines default: return nullptr; 6029dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner 60304d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov // vec_ld, vec_lvsl, vec_lvsr 60314d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvx: 60324d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvxl: 60334d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvebx: 60344d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvehx: 60354d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvewx: 60364d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvsl: 60374d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvsr: 6038176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case PPC::BI__builtin_vsx_lxvd2x: 6039176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case PPC::BI__builtin_vsx_lxvw4x: 60404d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov { 6041d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy); 60424d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov 6043578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer Ops[0] = Builder.CreateGEP(Ops[1], Ops[0]); 60444d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov Ops.pop_back(); 60454d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov 60464d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov switch (BuiltinID) { 6047b219cfc4d75f0a03630b7c4509ef791b7e97b2c8David Blaikie default: llvm_unreachable("Unsupported ld/lvsl/lvsr intrinsic!"); 60484d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvx: 60494d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov ID = Intrinsic::ppc_altivec_lvx; 60504d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov break; 60514d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvxl: 60524d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov ID = Intrinsic::ppc_altivec_lvxl; 60534d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov break; 60544d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvebx: 60554d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov ID = Intrinsic::ppc_altivec_lvebx; 60564d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov break; 60574d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvehx: 60584d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov ID = Intrinsic::ppc_altivec_lvehx; 60594d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov break; 60604d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvewx: 60614d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov ID = Intrinsic::ppc_altivec_lvewx; 60624d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov break; 60634d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvsl: 60644d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov ID = Intrinsic::ppc_altivec_lvsl; 60654d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov break; 60664d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvsr: 60674d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov ID = Intrinsic::ppc_altivec_lvsr; 60684d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov break; 6069176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case PPC::BI__builtin_vsx_lxvd2x: 6070176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines ID = Intrinsic::ppc_vsx_lxvd2x; 6071176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines break; 6072176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case PPC::BI__builtin_vsx_lxvw4x: 6073176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines ID = Intrinsic::ppc_vsx_lxvw4x; 6074176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines break; 60754d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov } 60764d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov llvm::Function *F = CGM.getIntrinsic(ID); 60774c7d9f1507d0f102bd4133bba63348636facd469Jay Foad return Builder.CreateCall(F, Ops, ""); 60784d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov } 60794d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov 6080dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner // vec_st 6081dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvx: 6082dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvxl: 6083dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvebx: 6084dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvehx: 6085dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvewx: 6086176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case PPC::BI__builtin_vsx_stxvd2x: 6087176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case PPC::BI__builtin_vsx_stxvw4x: 6088dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner { 6089d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy); 6090578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer Ops[1] = Builder.CreateGEP(Ops[2], Ops[1]); 6091dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner Ops.pop_back(); 6092dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner 6093dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner switch (BuiltinID) { 6094b219cfc4d75f0a03630b7c4509ef791b7e97b2c8David Blaikie default: llvm_unreachable("Unsupported st intrinsic!"); 6095dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvx: 6096dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner ID = Intrinsic::ppc_altivec_stvx; 6097dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner break; 6098dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvxl: 6099dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner ID = Intrinsic::ppc_altivec_stvxl; 6100dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner break; 6101dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvebx: 6102dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner ID = Intrinsic::ppc_altivec_stvebx; 6103dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner break; 6104dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvehx: 6105dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner ID = Intrinsic::ppc_altivec_stvehx; 6106dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner break; 6107dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvewx: 6108dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner ID = Intrinsic::ppc_altivec_stvewx; 6109dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner break; 6110176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case PPC::BI__builtin_vsx_stxvd2x: 6111176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines ID = Intrinsic::ppc_vsx_stxvd2x; 6112176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines break; 6113176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case PPC::BI__builtin_vsx_stxvw4x: 6114176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines ID = Intrinsic::ppc_vsx_stxvw4x; 6115176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines break; 6116dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner } 6117dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner llvm::Function *F = CGM.getIntrinsic(ID); 61184c7d9f1507d0f102bd4133bba63348636facd469Jay Foad return Builder.CreateCall(F, Ops, ""); 6119dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner } 6120dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner } 61211eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump} 6122c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 6123176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines// Emit an intrinsic that has 1 float or double. 6124176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hinesstatic Value *emitUnaryFPBuiltin(CodeGenFunction &CGF, 6125176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines const CallExpr *E, 6126176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines unsigned IntrinsicID) { 6127176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); 6128176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 6129176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); 6130176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return CGF.Builder.CreateCall(F, Src0); 6131176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines} 6132176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 6133176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines// Emit an intrinsic that has 3 float or double operands. 6134176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hinesstatic Value *emitTernaryFPBuiltin(CodeGenFunction &CGF, 6135176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines const CallExpr *E, 6136176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines unsigned IntrinsicID) { 6137176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); 6138176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1)); 6139176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2)); 6140176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 6141176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); 6142176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return CGF.Builder.CreateCall3(F, Src0, Src1, Src2); 6143176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines} 6144176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 6145176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines// Emit an intrinsic that has 1 float or double operand, and 1 integer. 6146176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hinesstatic Value *emitFPIntBuiltin(CodeGenFunction &CGF, 6147176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines const CallExpr *E, 6148176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines unsigned IntrinsicID) { 6149176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); 6150176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1)); 6151176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 6152176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); 6153176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return CGF.Builder.CreateCall2(F, Src0, Src1); 6154176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines} 6155176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 6156c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen HinesValue *CodeGenFunction::EmitR600BuiltinExpr(unsigned BuiltinID, 6157c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines const CallExpr *E) { 6158c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines switch (BuiltinID) { 6159c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case R600::BI__builtin_amdgpu_div_scale: 6160c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case R600::BI__builtin_amdgpu_div_scalef: { 6161c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines // Translate from the intrinsics's struct return to the builtin's out 6162c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines // argument. 6163c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 6164c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines std::pair<llvm::Value *, unsigned> FlagOutPtr 6165c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines = EmitPointerWithAlignment(E->getArg(3)); 6166c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 6167c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *X = EmitScalarExpr(E->getArg(0)); 6168c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Y = EmitScalarExpr(E->getArg(1)); 6169c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Z = EmitScalarExpr(E->getArg(2)); 6170c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 6171c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Callee = CGM.getIntrinsic(Intrinsic::AMDGPU_div_scale, 6172c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines X->getType()); 6173c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 6174c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Tmp = Builder.CreateCall3(Callee, X, Y, Z); 6175c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 6176c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Result = Builder.CreateExtractValue(Tmp, 0); 6177c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Flag = Builder.CreateExtractValue(Tmp, 1); 6178c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 6179c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Type *RealFlagType 6180c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines = FlagOutPtr.first->getType()->getPointerElementType(); 6181c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 6182c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *FlagExt = Builder.CreateZExt(Flag, RealFlagType); 6183c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::StoreInst *FlagStore = Builder.CreateStore(FlagExt, FlagOutPtr.first); 6184c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines FlagStore->setAlignment(FlagOutPtr.second); 6185c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines return Result; 6186176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 6187176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case R600::BI__builtin_amdgpu_div_fmas: 6188176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case R600::BI__builtin_amdgpu_div_fmasf: { 6189176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::Value *Src0 = EmitScalarExpr(E->getArg(0)); 6190176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::Value *Src1 = EmitScalarExpr(E->getArg(1)); 6191176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::Value *Src2 = EmitScalarExpr(E->getArg(2)); 6192176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::Value *Src3 = EmitScalarExpr(E->getArg(3)); 6193176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 6194176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::Value *F = CGM.getIntrinsic(Intrinsic::AMDGPU_div_fmas, 6195176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Src0->getType()); 6196176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::Value *Src3ToBool = Builder.CreateIsNotNull(Src3); 6197176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return Builder.CreateCall4(F, Src0, Src1, Src2, Src3ToBool); 6198176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 6199176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case R600::BI__builtin_amdgpu_div_fixup: 6200176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case R600::BI__builtin_amdgpu_div_fixupf: 6201176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return emitTernaryFPBuiltin(*this, E, Intrinsic::AMDGPU_div_fixup); 6202176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case R600::BI__builtin_amdgpu_trig_preop: 6203176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case R600::BI__builtin_amdgpu_trig_preopf: 6204176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return emitFPIntBuiltin(*this, E, Intrinsic::AMDGPU_trig_preop); 6205176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case R600::BI__builtin_amdgpu_rcp: 6206176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case R600::BI__builtin_amdgpu_rcpf: 6207176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return emitUnaryFPBuiltin(*this, E, Intrinsic::AMDGPU_rcp); 6208176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case R600::BI__builtin_amdgpu_rsq: 6209176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case R600::BI__builtin_amdgpu_rsqf: 6210176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return emitUnaryFPBuiltin(*this, E, Intrinsic::AMDGPU_rsq); 6211176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case R600::BI__builtin_amdgpu_rsq_clamped: 6212176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case R600::BI__builtin_amdgpu_rsq_clampedf: 6213176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return emitUnaryFPBuiltin(*this, E, Intrinsic::AMDGPU_rsq_clamped); 6214176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case R600::BI__builtin_amdgpu_ldexp: 6215176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case R600::BI__builtin_amdgpu_ldexpf: 6216176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return emitFPIntBuiltin(*this, E, Intrinsic::AMDGPU_ldexp); 6217176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines default: 6218c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines return nullptr; 6219c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines } 6220c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines} 6221