1022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson//===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===// 2022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson// 3022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson// The LLVM Compiler Infrastructure 4022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson// 50bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// This file is distributed under the University of Illinois Open Source 60bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// License. See LICENSE.TXT for details. 7022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson// 8022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson//===----------------------------------------------------------------------===// 9022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson// 10022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson// This contains code to emit Builtin calls as LLVM code. 11022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson// 12022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson//===----------------------------------------------------------------------===// 13022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson 14022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlsson#include "CodeGenFunction.h" 153ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar#include "CGCXXABI.h" 1655bcace250e1ff366e4482714b344b8cbc8be5f3Fariborz Jahanian#include "CGObjCRuntime.h" 1755fc873017f10f6f566b182b70f6fc22aefa3464Chandler Carruth#include "CodeGenModule.h" 1855fc873017f10f6f566b182b70f6fc22aefa3464Chandler Carruth#include "TargetInfo.h" 19bef20ac367a09555b30d6eb3847a81ec164caf88Chris Lattner#include "clang/AST/ASTContext.h" 20c4a1dea2dc56bd1357ec91b829a0b9e68229a13eDaniel Dunbar#include "clang/AST/Decl.h" 216b15cdc1312f8fc45c86ee75e2a85106700e97f6Chris Lattner#include "clang/Basic/TargetBuiltins.h" 2255fc873017f10f6f566b182b70f6fc22aefa3464Chandler Carruth#include "clang/Basic/TargetInfo.h" 238b54999a831bb195c08541ca995ef0505c96193fMark Lacey#include "clang/CodeGen/CGFunctionInfo.h" 240e2c34f92f00628d48968dfea096d36381f494cbStephen Hines#include "llvm/ADT/StringExtras.h" 250e2c34f92f00628d48968dfea096d36381f494cbStephen Hines#include "llvm/IR/CallSite.h" 263b844ba7d5be205a9b4f5f0b0d1b7978977f4b8cChandler Carruth#include "llvm/IR/DataLayout.h" 270e2c34f92f00628d48968dfea096d36381f494cbStephen Hines#include "llvm/IR/InlineAsm.h" 283b844ba7d5be205a9b4f5f0b0d1b7978977f4b8cChandler Carruth#include "llvm/IR/Intrinsics.h" 294967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar#include "llvm/IR/MDBuilder.h" 303ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar#include <sstream> 31558229f267d2cdee69639df04382082e7c7ad54cJakub Staszak 32022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlssonusing namespace clang; 33022012e6e5626c3372e1a5493c0929dfc1fa9e47Anders Carlssonusing namespace CodeGen; 34ca6fcfad547dcec3fdd17790b4fab0918df74b37Anders Carlssonusing namespace llvm; 35ca6fcfad547dcec3fdd17790b4fab0918df74b37Anders Carlsson 36a45680b7e7c49ea9893c6cff585984f3e4120366John McCall/// getBuiltinLibFunction - Given a builtin id for a function like 37a45680b7e7c49ea9893c6cff585984f3e4120366John McCall/// "__builtin_fabsf", return a Function* for "fabsf". 38a45680b7e7c49ea9893c6cff585984f3e4120366John McCallllvm::Value *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD, 39a45680b7e7c49ea9893c6cff585984f3e4120366John McCall unsigned BuiltinID) { 40a45680b7e7c49ea9893c6cff585984f3e4120366John McCall assert(Context.BuiltinInfo.isLibFunction(BuiltinID)); 41a45680b7e7c49ea9893c6cff585984f3e4120366John McCall 42a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // Get the name, skip over the __builtin_ prefix (if necessary). 43a45680b7e7c49ea9893c6cff585984f3e4120366John McCall StringRef Name; 44a45680b7e7c49ea9893c6cff585984f3e4120366John McCall GlobalDecl D(FD); 45a45680b7e7c49ea9893c6cff585984f3e4120366John McCall 46a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // If the builtin has been declared explicitly with an assembler label, 47a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // use the mangled name. This differs from the plain label on platforms 48a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // that prefix labels. 49a45680b7e7c49ea9893c6cff585984f3e4120366John McCall if (FD->hasAttr<AsmLabelAttr>()) 50a45680b7e7c49ea9893c6cff585984f3e4120366John McCall Name = getMangledName(D); 51a45680b7e7c49ea9893c6cff585984f3e4120366John McCall else 5287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Name = Context.BuiltinInfo.getName(BuiltinID) + 10; 53a45680b7e7c49ea9893c6cff585984f3e4120366John McCall 54a45680b7e7c49ea9893c6cff585984f3e4120366John McCall llvm::FunctionType *Ty = 55a45680b7e7c49ea9893c6cff585984f3e4120366John McCall cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType())); 56a45680b7e7c49ea9893c6cff585984f3e4120366John McCall 57a45680b7e7c49ea9893c6cff585984f3e4120366John McCall return GetOrCreateLLVMFunction(Name, Ty, D, /*ForVTable=*/false); 58a45680b7e7c49ea9893c6cff585984f3e4120366John McCall} 59a45680b7e7c49ea9893c6cff585984f3e4120366John McCall 6026815d97c5743481e317f17a8d53a6819d061862John McCall/// Emit the conversions required to turn the given value into an 6126815d97c5743481e317f17a8d53a6819d061862John McCall/// integer of the given size. 6226815d97c5743481e317f17a8d53a6819d061862John McCallstatic Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V, 632acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner QualType T, llvm::IntegerType *IntType) { 6426815d97c5743481e317f17a8d53a6819d061862John McCall V = CGF.EmitToMemory(V, T); 6526815d97c5743481e317f17a8d53a6819d061862John McCall 6626815d97c5743481e317f17a8d53a6819d061862John McCall if (V->getType()->isPointerTy()) 6726815d97c5743481e317f17a8d53a6819d061862John McCall return CGF.Builder.CreatePtrToInt(V, IntType); 6826815d97c5743481e317f17a8d53a6819d061862John McCall 6926815d97c5743481e317f17a8d53a6819d061862John McCall assert(V->getType() == IntType); 7026815d97c5743481e317f17a8d53a6819d061862John McCall return V; 71db4325b098eff5e9e660db19f0148423fb21f27fChandler Carruth} 72db4325b098eff5e9e660db19f0148423fb21f27fChandler Carruth 7326815d97c5743481e317f17a8d53a6819d061862John McCallstatic Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V, 742acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner QualType T, llvm::Type *ResultType) { 7526815d97c5743481e317f17a8d53a6819d061862John McCall V = CGF.EmitFromMemory(V, T); 7626815d97c5743481e317f17a8d53a6819d061862John McCall 7726815d97c5743481e317f17a8d53a6819d061862John McCall if (ResultType->isPointerTy()) 7826815d97c5743481e317f17a8d53a6819d061862John McCall return CGF.Builder.CreateIntToPtr(V, ResultType); 7926815d97c5743481e317f17a8d53a6819d061862John McCall 8026815d97c5743481e317f17a8d53a6819d061862John McCall assert(V->getType() == ResultType); 8126815d97c5743481e317f17a8d53a6819d061862John McCall return V; 82db4325b098eff5e9e660db19f0148423fb21f27fChandler Carruth} 83db4325b098eff5e9e660db19f0148423fb21f27fChandler Carruth 840002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar/// Utility to insert an atomic instruction based on Instrinsic::ID 850002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar/// and the expression node. 8687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic Value *MakeBinaryAtomicValue(CodeGenFunction &CGF, 8787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::AtomicRMWInst::BinOp Kind, 8887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar const CallExpr *E) { 8926815d97c5743481e317f17a8d53a6819d061862John McCall QualType T = E->getType(); 9026815d97c5743481e317f17a8d53a6819d061862John McCall assert(E->getArg(0)->getType()->isPointerType()); 9126815d97c5743481e317f17a8d53a6819d061862John McCall assert(CGF.getContext().hasSameUnqualifiedType(T, 9226815d97c5743481e317f17a8d53a6819d061862John McCall E->getArg(0)->getType()->getPointeeType())); 9326815d97c5743481e317f17a8d53a6819d061862John McCall assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType())); 9426815d97c5743481e317f17a8d53a6819d061862John McCall 954f209445c06a43283c6f72dda7c925538b1578e9Chris Lattner llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0)); 96956a5a17713deb1b5b27893303c4f308a1bd2a62Micah Villmow unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); 9726815d97c5743481e317f17a8d53a6819d061862John McCall 989cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::IntegerType *IntType = 99db4325b098eff5e9e660db19f0148423fb21f27fChandler Carruth llvm::IntegerType::get(CGF.getLLVMContext(), 10026815d97c5743481e317f17a8d53a6819d061862John McCall CGF.getContext().getTypeSize(T)); 1019cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); 10226815d97c5743481e317f17a8d53a6819d061862John McCall 10326815d97c5743481e317f17a8d53a6819d061862John McCall llvm::Value *Args[2]; 10426815d97c5743481e317f17a8d53a6819d061862John McCall Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType); 10526815d97c5743481e317f17a8d53a6819d061862John McCall Args[1] = CGF.EmitScalarExpr(E->getArg(1)); 1062acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ValueType = Args[1]->getType(); 10726815d97c5743481e317f17a8d53a6819d061862John McCall Args[1] = EmitToInt(CGF, Args[1], T, IntType); 10826815d97c5743481e317f17a8d53a6819d061862John McCall 1094967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Value *Result = CGF.Builder.CreateAtomicRMW( 1104967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent); 11187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return EmitFromInt(CGF, Result, T, ValueType); 11287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 11387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 11487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic Value *EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E) { 11587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Val = CGF.EmitScalarExpr(E->getArg(0)); 11687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Address = CGF.EmitScalarExpr(E->getArg(1)); 11787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 11887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Convert the type of the pointer to a pointer to the stored type. 11987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Val = CGF.EmitToMemory(Val, E->getArg(0)->getType()); 12087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *BC = CGF.Builder.CreateBitCast( 12187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address, llvm::PointerType::getUnqual(Val->getType()), "cast"); 12287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar LValue LV = CGF.MakeNaturalAlignAddrLValue(BC, E->getArg(0)->getType()); 12387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar LV.setNontemporal(true); 12487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar CGF.EmitStoreOfScalar(Val, LV, false); 12587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return nullptr; 12687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 12787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 12887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic Value *EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E) { 12987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Address = CGF.EmitScalarExpr(E->getArg(0)); 13087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 13187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar LValue LV = CGF.MakeNaturalAlignAddrLValue(Address, E->getType()); 13287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar LV.setNontemporal(true); 13387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return CGF.EmitLoadOfScalar(LV, E->getExprLoc()); 13487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 13587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 13687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic RValue EmitBinaryAtomic(CodeGenFunction &CGF, 13787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::AtomicRMWInst::BinOp Kind, 13887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar const CallExpr *E) { 13987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return RValue::get(MakeBinaryAtomicValue(CGF, Kind, E)); 1400002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar} 1410002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar 1420002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar/// Utility to insert an atomic instruction based Instrinsic::ID and 14326815d97c5743481e317f17a8d53a6819d061862John McCall/// the expression node, where the return value is the result of the 14426815d97c5743481e317f17a8d53a6819d061862John McCall/// operation. 145420b11850d3f4557421f43f519b59d528329c668Chris Lattnerstatic RValue EmitBinaryAtomicPost(CodeGenFunction &CGF, 146c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman llvm::AtomicRMWInst::BinOp Kind, 147c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman const CallExpr *E, 148176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Instruction::BinaryOps Op, 149176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines bool Invert = false) { 15026815d97c5743481e317f17a8d53a6819d061862John McCall QualType T = E->getType(); 15126815d97c5743481e317f17a8d53a6819d061862John McCall assert(E->getArg(0)->getType()->isPointerType()); 15226815d97c5743481e317f17a8d53a6819d061862John McCall assert(CGF.getContext().hasSameUnqualifiedType(T, 15326815d97c5743481e317f17a8d53a6819d061862John McCall E->getArg(0)->getType()->getPointeeType())); 15426815d97c5743481e317f17a8d53a6819d061862John McCall assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType())); 15526815d97c5743481e317f17a8d53a6819d061862John McCall 1564f209445c06a43283c6f72dda7c925538b1578e9Chris Lattner llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0)); 157956a5a17713deb1b5b27893303c4f308a1bd2a62Micah Villmow unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); 15826815d97c5743481e317f17a8d53a6819d061862John McCall 1599cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::IntegerType *IntType = 160db4325b098eff5e9e660db19f0148423fb21f27fChandler Carruth llvm::IntegerType::get(CGF.getLLVMContext(), 16126815d97c5743481e317f17a8d53a6819d061862John McCall CGF.getContext().getTypeSize(T)); 1629cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); 16326815d97c5743481e317f17a8d53a6819d061862John McCall 16426815d97c5743481e317f17a8d53a6819d061862John McCall llvm::Value *Args[2]; 16526815d97c5743481e317f17a8d53a6819d061862John McCall Args[1] = CGF.EmitScalarExpr(E->getArg(1)); 1662acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ValueType = Args[1]->getType(); 16726815d97c5743481e317f17a8d53a6819d061862John McCall Args[1] = EmitToInt(CGF, Args[1], T, IntType); 16826815d97c5743481e317f17a8d53a6819d061862John McCall Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType); 16926815d97c5743481e317f17a8d53a6819d061862John McCall 1704967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Value *Result = CGF.Builder.CreateAtomicRMW( 1714967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent); 17226815d97c5743481e317f17a8d53a6819d061862John McCall Result = CGF.Builder.CreateBinOp(Op, Result, Args[1]); 173176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (Invert) 174176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Result = CGF.Builder.CreateBinOp(llvm::Instruction::Xor, Result, 175176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::ConstantInt::get(IntType, -1)); 17626815d97c5743481e317f17a8d53a6819d061862John McCall Result = EmitFromInt(CGF, Result, T, ValueType); 17726815d97c5743481e317f17a8d53a6819d061862John McCall return RValue::get(Result); 1781ffe281890f3cd7728316b45a1f3dd4d3120af7bMon P Wang} 1791ffe281890f3cd7728316b45a1f3dd4d3120af7bMon P Wang 18087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar/// @brief Utility to insert an atomic cmpxchg instruction. 18187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar/// 18287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar/// @param CGF The current codegen function. 18387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar/// @param E Builtin call expression to convert to cmpxchg. 18487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar/// arg0 - address to operate on 18587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar/// arg1 - value to compare with 18687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar/// arg2 - new value 18787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar/// @param ReturnBool Specifies whether to return success flag of 18887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar/// cmpxchg result or the old value. 18987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar/// 19087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar/// @returns result of cmpxchg, according to ReturnBool 19187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic Value *MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E, 19287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar bool ReturnBool) { 19387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar QualType T = ReturnBool ? E->getArg(1)->getType() : E->getType(); 19487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0)); 19587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); 19687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 19787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::IntegerType *IntType = llvm::IntegerType::get( 19887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar CGF.getLLVMContext(), CGF.getContext().getTypeSize(T)); 19987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); 20087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 20187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Args[3]; 20287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType); 20387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Args[1] = CGF.EmitScalarExpr(E->getArg(1)); 20487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *ValueType = Args[1]->getType(); 20587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Args[1] = EmitToInt(CGF, Args[1], T, IntType); 20687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Args[2] = EmitToInt(CGF, CGF.EmitScalarExpr(E->getArg(2)), T, IntType); 20787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 2084967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Pair = CGF.Builder.CreateAtomicCmpXchg( 2094967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Args[0], Args[1], Args[2], llvm::AtomicOrdering::SequentiallyConsistent, 2104967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering::SequentiallyConsistent); 21187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar if (ReturnBool) 21287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Extract boolean success flag and zext it to int. 21387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return CGF.Builder.CreateZExt(CGF.Builder.CreateExtractValue(Pair, 1), 21487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar CGF.ConvertType(E->getType())); 21587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar else 21687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Extract old value and emit it using the same type as compare value. 21787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return EmitFromInt(CGF, CGF.Builder.CreateExtractValue(Pair, 0), T, 21887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar ValueType); 21987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 22087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 2214967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar// Emit a simple mangled intrinsic that has 1 argument and a return type 2224967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar// matching the argument type. 2234967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainarstatic Value *emitUnaryBuiltin(CodeGenFunction &CGF, 2244967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar const CallExpr *E, 2254967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar unsigned IntrinsicID) { 2264967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); 2274967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 2284967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); 2294967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return CGF.Builder.CreateCall(F, Src0); 2304967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar} 2314967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 2324967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar// Emit an intrinsic that has 2 operands of the same type as its result. 2334967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainarstatic Value *emitBinaryBuiltin(CodeGenFunction &CGF, 2344967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar const CallExpr *E, 2354967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar unsigned IntrinsicID) { 2364967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); 2374967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1)); 2384967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 2394967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); 2404967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return CGF.Builder.CreateCall(F, { Src0, Src1 }); 2414967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar} 2424967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 2434967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar// Emit an intrinsic that has 3 operands of the same type as its result. 2444967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainarstatic Value *emitTernaryBuiltin(CodeGenFunction &CGF, 2454967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar const CallExpr *E, 2464967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar unsigned IntrinsicID) { 2474967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); 2484967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1)); 2494967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2)); 2504967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 2514967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); 2524967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return CGF.Builder.CreateCall(F, { Src0, Src1, Src2 }); 2534967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar} 2544967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 2554967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar// Emit an intrinsic that has 1 float or double operand, and 1 integer. 2564967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainarstatic Value *emitFPIntBuiltin(CodeGenFunction &CGF, 2574967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar const CallExpr *E, 2584967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar unsigned IntrinsicID) { 2594967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); 2604967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1)); 2614967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 2624967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); 2634967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return CGF.Builder.CreateCall(F, {Src0, Src1}); 2644967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar} 2654967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 266176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines/// EmitFAbs - Emit a call to @llvm.fabs(). 267176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hinesstatic Value *EmitFAbs(CodeGenFunction &CGF, Value *V) { 268176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *F = CGF.CGM.getIntrinsic(Intrinsic::fabs, V->getType()); 269176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::CallInst *Call = CGF.Builder.CreateCall(F, V); 270176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Call->setDoesNotAccessMemory(); 271176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return Call; 272420b11850d3f4557421f43f519b59d528329c668Chris Lattner} 273420b11850d3f4557421f43f519b59d528329c668Chris Lattner 2743ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar/// Emit the computation of the sign bit for a floating point value. Returns 2753ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar/// the i1 sign bit value. 2763ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainarstatic Value *EmitSignBit(CodeGenFunction &CGF, Value *V) { 2773ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar LLVMContext &C = CGF.CGM.getLLVMContext(); 2783ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 2793ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar llvm::Type *Ty = V->getType(); 2803ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar int Width = Ty->getPrimitiveSizeInBits(); 2813ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar llvm::Type *IntTy = llvm::IntegerType::get(C, Width); 2823ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar V = CGF.Builder.CreateBitCast(V, IntTy); 2833ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (Ty->isPPC_FP128Ty()) { 28487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // We want the sign bit of the higher-order double. The bitcast we just 28587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // did works as if the double-double was stored to memory and then 28687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // read as an i128. The "store" will put the higher-order double in the 28787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // lower address in both little- and big-Endian modes, but the "load" 28887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // will treat those bits as a different part of the i128: the low bits in 28987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // little-Endian, the high bits in big-Endian. Therefore, on big-Endian 29087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // we need to shift the high bits down to the low before truncating. 2913ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Width >>= 1; 29287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar if (CGF.getTarget().isBigEndian()) { 29387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width); 29487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar V = CGF.Builder.CreateLShr(V, ShiftCst); 2954967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 2964967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // We are truncating value in order to extract the higher-order 29787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // double, which we will be using to extract the sign from. 2983ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar IntTy = llvm::IntegerType::get(C, Width); 2993ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar V = CGF.Builder.CreateTrunc(V, IntTy); 3003ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar } 3013ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Value *Zero = llvm::Constant::getNullValue(IntTy); 3023ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar return CGF.Builder.CreateICmpSLT(V, Zero); 3033ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar} 3043ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 305a45680b7e7c49ea9893c6cff585984f3e4120366John McCallstatic RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *Fn, 306a45680b7e7c49ea9893c6cff585984f3e4120366John McCall const CallExpr *E, llvm::Value *calleeValue) { 307176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return CGF.EmitCall(E->getCallee()->getType(), calleeValue, E, 308176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines ReturnValueSlot(), Fn); 309a45680b7e7c49ea9893c6cff585984f3e4120366John McCall} 310a45680b7e7c49ea9893c6cff585984f3e4120366John McCall 3110cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// \brief Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.* 3120cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// depending on IntrinsicID. 3130cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// 3140cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// \arg CGF The current codegen function. 3150cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// \arg IntrinsicID The ID for the Intrinsic we wish to generate. 3160cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// \arg X The first argument to the llvm.*.with.overflow.*. 3170cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// \arg Y The second argument to the llvm.*.with.overflow.*. 3180cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// \arg Carry The carry returned by the llvm.*.with.overflow.*. 3190cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman/// \returns The result (i.e. sum/product) returned by the intrinsic. 3200cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesmanstatic llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF, 3210cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman const llvm::Intrinsic::ID IntrinsicID, 3220cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *X, llvm::Value *Y, 3230cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *&Carry) { 3240cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // Make sure we have integers of the same width. 3250cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman assert(X->getType() == Y->getType() && 3260cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman "Arguments must be the same type. (Did you forget to make sure both " 3270cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman "arguments have the same integer width?)"); 3280cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman 329563fb903fab15972e2d9c66b0ae046a94be86a71NAKAMURA Takumi llvm::Value *Callee = CGF.CGM.getIntrinsic(IntrinsicID, X->getType()); 330b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Value *Tmp = CGF.Builder.CreateCall(Callee, {X, Y}); 3310cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman Carry = CGF.Builder.CreateExtractValue(Tmp, 1); 3320cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman return CGF.Builder.CreateExtractValue(Tmp, 0); 3330cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman} 3340cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman 3354967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainarstatic Value *emitRangedBuiltin(CodeGenFunction &CGF, 3364967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar unsigned IntrinsicID, 3374967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar int low, int high) { 3384967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::MDBuilder MDHelper(CGF.getLLVMContext()); 3394967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::MDNode *RNode = MDHelper.createRange(APInt(32, low), APInt(32, high)); 3404967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *F = CGF.CGM.getIntrinsic(IntrinsicID, {}); 3414967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Instruction *Call = CGF.Builder.CreateCall(F); 3424967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Call->setMetadata(llvm::LLVMContext::MD_range, RNode); 3434967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return Call; 3444967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar} 3454967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 34687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarnamespace { 34787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar struct WidthAndSignedness { 34887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar unsigned Width; 34987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar bool Signed; 35087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar }; 35187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 35287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 35387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic WidthAndSignedness 35487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga NainargetIntegerWidthAndSignedness(const clang::ASTContext &context, 35587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar const clang::QualType Type) { 35687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar assert(Type->isIntegerType() && "Given type is not an integer."); 35787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar unsigned Width = Type->isBooleanType() ? 1 : context.getTypeInfo(Type).Width; 35887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar bool Signed = Type->isSignedIntegerType(); 35987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return {Width, Signed}; 36087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 36187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 36287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar// Given one or more integer types, this function produces an integer type that 36387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar// encompasses them: any value in one of the given types could be expressed in 36487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar// the encompassing type. 36587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic struct WidthAndSignedness 36687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga NainarEncompassingIntegerType(ArrayRef<struct WidthAndSignedness> Types) { 36787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar assert(Types.size() > 0 && "Empty list of types."); 36887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 36987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // If any of the given types is signed, we must return a signed type. 37087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar bool Signed = false; 37187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar for (const auto &Type : Types) { 37287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Signed |= Type.Signed; 37387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 37487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 37587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // The encompassing type must have a width greater than or equal to the width 37687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // of the specified types. Aditionally, if the encompassing type is signed, 37787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // its width must be strictly greater than the width of any unsigned types 37887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // given. 37987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar unsigned Width = 0; 38087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar for (const auto &Type : Types) { 38187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar unsigned MinWidth = Type.Width + (Signed && !Type.Signed); 38287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar if (Width < MinWidth) { 38387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Width = MinWidth; 38487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 38587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 38687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 38787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return {Width, Signed}; 38887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 38987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 39087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga NainarValue *CodeGenFunction::EmitVAStartEnd(Value *ArgValue, bool IsStart) { 39187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *DestType = Int8PtrTy; 39287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar if (ArgValue->getType() != DestType) 39387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar ArgValue = 39487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateBitCast(ArgValue, DestType, ArgValue->getName().data()); 39587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 39687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend; 39787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue); 39887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 39987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 40087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar/// Checks if using the result of __builtin_object_size(p, @p From) in place of 40187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar/// __builtin_object_size(p, @p To) is correct 40287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic bool areBOSTypesCompatible(int From, int To) { 40387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Note: Our __builtin_object_size implementation currently treats Type=0 and 40487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Type=2 identically. Encoding this implementation detail here may make 40587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // improving __builtin_object_size difficult in the future, so it's omitted. 40687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return From == To || (From == 0 && To == 1) || (From == 3 && To == 2); 40787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 40887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 40987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic llvm::Value * 41087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga NainargetDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType) { 41187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return ConstantInt::get(ResType, (Type & 2) ? 0 : -1, /*isSigned=*/true); 41287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 41387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 41487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarllvm::Value * 41587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga NainarCodeGenFunction::evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type, 41687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::IntegerType *ResType) { 41787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar uint64_t ObjectSize; 41887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar if (!E->tryEvaluateObjectSize(ObjectSize, getContext(), Type)) 41987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return emitBuiltinObjectSize(E, Type, ResType); 42087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return ConstantInt::get(ResType, ObjectSize, /*isSigned=*/true); 42187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 42287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 42387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar/// Returns a Value corresponding to the size of the given expression. 42487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar/// This Value may be either of the following: 42587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar/// - A llvm::Argument (if E is a param with the pass_object_size attribute on 42687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar/// it) 42787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar/// - A call to the @llvm.objectsize intrinsic 42887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarllvm::Value * 42987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga NainarCodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type, 43087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::IntegerType *ResType) { 43187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // We need to reference an argument if the pointer is a parameter with the 43287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // pass_object_size attribute. 43387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar if (auto *D = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts())) { 43487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar auto *Param = dyn_cast<ParmVarDecl>(D->getDecl()); 43587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>(); 43687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar if (Param != nullptr && PS != nullptr && 43787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar areBOSTypesCompatible(PS->getType(), Type)) { 43887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar auto Iter = SizeArguments.find(Param); 43987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar assert(Iter != SizeArguments.end()); 44087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 44187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar const ImplicitParamDecl *D = Iter->second; 44287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar auto DIter = LocalDeclMap.find(D); 44387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar assert(DIter != LocalDeclMap.end()); 44487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 44587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return EmitLoadOfScalar(DIter->second, /*volatile=*/false, 44687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar getContext().getSizeType(), E->getLocStart()); 44787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 44887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 44987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 45087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // LLVM can't handle Type=3 appropriately, and __builtin_object_size shouldn't 45187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // evaluate E for side-effects. In either case, we shouldn't lower to 45287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // @llvm.objectsize. 45387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar if (Type == 3 || E->HasSideEffects(getContext())) 45487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return getDefaultBuiltinObjectSizeResult(Type, ResType); 45587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 45687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // LLVM only supports 0 and 2, make sure that we pass along that 45787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // as a boolean. 45887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar auto *CI = ConstantInt::get(Builder.getInt1Ty(), (Type & 2) >> 1); 45987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // FIXME: Get right address space. 46087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *Tys[] = {ResType, Builder.getInt8PtrTy(0)}; 46187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *F = CGM.getIntrinsic(Intrinsic::objectsize, Tys); 46287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateCall(F, {EmitScalarExpr(E), CI}); 46387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 46487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 4651eb4433ac451dc16f4133a88af2d002ac26c58efMike StumpRValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, 4660e2c34f92f00628d48968dfea096d36381f494cbStephen Hines unsigned BuiltinID, const CallExpr *E, 4670e2c34f92f00628d48968dfea096d36381f494cbStephen Hines ReturnValueSlot ReturnValue) { 468564ea2a99b3afeac9ded332730a56db1f6358a58Chris Lattner // See if we can constant fold this builtin. If so, don't emit it at all. 469f35d35a2316dcb65d078844696c2032b71a7f103Anders Carlsson Expr::EvalResult Result; 47052a27f5be98d99fd5339949d8a3d0b2aec655e0bEli Friedman if (E->EvaluateAsRValue(Result, CGM.getContext()) && 471dd697bc8629f0fa6777245610d52ca7ceb3b67f4Fariborz Jahanian !Result.hasSideEffects()) { 472f35d35a2316dcb65d078844696c2032b71a7f103Anders Carlsson if (Result.Val.isInt()) 473d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall return RValue::get(llvm::ConstantInt::get(getLLVMContext(), 4744a28d5deeba33722aa009eab488591fb9055cc7eOwen Anderson Result.Val.getInt())); 475a1aa9e36e6e21f74c56cf9e72cb5bd9aa2a92fd4Chris Lattner if (Result.Val.isFloat()) 476d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall return RValue::get(llvm::ConstantFP::get(getLLVMContext(), 477d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Result.Val.getFloat())); 4781f32999ec79a980576e100d64d5f3267eb19ea49Chris Lattner } 4791eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 480564ea2a99b3afeac9ded332730a56db1f6358a58Chris Lattner switch (BuiltinID) { 481564ea2a99b3afeac9ded332730a56db1f6358a58Chris Lattner default: break; // Handle intrinsics and libm functions below. 482506ff88f44562df267b6a06608ab841b76df2a2bChris Lattner case Builtin::BI__builtin___CFStringMakeConstantString: 4830d13f6fdbdd6f06e2449b8834dda53334abd399aDavid Chisnall case Builtin::BI__builtin___NSStringMakeConstantString: 4846bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(CGM.EmitConstantExpr(E, E->getType(), nullptr)); 4856a705f0443f8398ece14d80e71d8e8c9e71aa84aChris Lattner case Builtin::BI__builtin_stdarg_start: 486793680ed8104bf088d1b382b963a8badcb3f07deAnders Carlsson case Builtin::BI__builtin_va_start: 487651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case Builtin::BI__va_start: 48887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case Builtin::BI__builtin_va_end: 48987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return RValue::get( 49087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar EmitVAStartEnd(BuiltinID == Builtin::BI__va_start 49187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar ? EmitScalarExpr(E->getArg(0)) 49287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar : EmitVAListRef(E->getArg(0)).getPointer(), 49387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar BuiltinID != Builtin::BI__builtin_va_end)); 494a28ef8b5c6ea452472d89041128fac8b683fe968Anders Carlsson case Builtin::BI__builtin_va_copy: { 49587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *DstPtr = EmitVAListRef(E->getArg(0)).getPointer(); 49687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *SrcPtr = EmitVAListRef(E->getArg(1)).getPointer(); 497a28ef8b5c6ea452472d89041128fac8b683fe968Anders Carlsson 4982acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *Type = Int8PtrTy; 499a28ef8b5c6ea452472d89041128fac8b683fe968Anders Carlsson 500a28ef8b5c6ea452472d89041128fac8b683fe968Anders Carlsson DstPtr = Builder.CreateBitCast(DstPtr, Type); 501a28ef8b5c6ea452472d89041128fac8b683fe968Anders Carlsson SrcPtr = Builder.CreateBitCast(SrcPtr, Type); 502b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return RValue::get(Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy), 503b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar {DstPtr, SrcPtr})); 504a28ef8b5c6ea452472d89041128fac8b683fe968Anders Carlsson } 505258f930227c1a102c9c22eee88df65f748863425Jim Grosbach case Builtin::BI__builtin_abs: 506f7b2d8b3b1971e73e2d2f0662520dc7693235a7fEli Friedman case Builtin::BI__builtin_labs: 507f7b2d8b3b1971e73e2d2f0662520dc7693235a7fEli Friedman case Builtin::BI__builtin_llabs: { 5081eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump Value *ArgValue = EmitScalarExpr(E->getArg(0)); 5091eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 5109a847f598c4907a72b8593b364b9e6b94b086e75Chris Lattner Value *NegOp = Builder.CreateNeg(ArgValue, "neg"); 5111eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump Value *CmpResult = 5121eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump Builder.CreateICmpSGE(ArgValue, 513c9c88b4159791c48e486ca94e3743b5979e2b7a6Owen Anderson llvm::Constant::getNullValue(ArgValue->getType()), 5149a847f598c4907a72b8593b364b9e6b94b086e75Chris Lattner "abscond"); 5151eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump Value *Result = 516c2251dc59b0edc28f9303637dec970a7520939adAnders Carlsson Builder.CreateSelect(CmpResult, ArgValue, NegOp, "abs"); 5171eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 518c2251dc59b0edc28f9303637dec970a7520939adAnders Carlsson return RValue::get(Result); 519c2251dc59b0edc28f9303637dec970a7520939adAnders Carlsson } 520176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__builtin_fabs: 521176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__builtin_fabsf: 522176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__builtin_fabsl: { 5234967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::fabs)); 524176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 525176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__builtin_fmod: 526176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__builtin_fmodf: 527176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__builtin_fmodl: { 528176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *Arg1 = EmitScalarExpr(E->getArg(0)); 529176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *Arg2 = EmitScalarExpr(E->getArg(1)); 530176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *Result = Builder.CreateFRem(Arg1, Arg2, "fmod"); 531176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return RValue::get(Result); 532176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 5334967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_copysign: 5344967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_copysignf: 5354967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_copysignl: { 5364967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get(emitBinaryBuiltin(*this, E, Intrinsic::copysign)); 5374967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 5384967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_ceil: 5394967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_ceilf: 5404967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_ceill: { 5414967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::ceil)); 5424967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 5434967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_floor: 5444967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_floorf: 5454967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_floorl: { 5464967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::floor)); 5474967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 5484967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_trunc: 5494967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_truncf: 5504967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_truncl: { 5514967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::trunc)); 5524967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 5534967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_rint: 5544967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_rintf: 5554967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_rintl: { 5564967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::rint)); 5574967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 5584967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_nearbyint: 5594967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_nearbyintf: 5604967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_nearbyintl: { 5614967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::nearbyint)); 5624967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 5634967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_round: 5644967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_roundf: 5654967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_roundl: { 5664967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::round)); 5674967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 5684967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_fmin: 5694967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_fminf: 5704967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_fminl: { 5714967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get(emitBinaryBuiltin(*this, E, Intrinsic::minnum)); 5724967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 5734967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_fmax: 5744967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_fmaxf: 5754967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_fmaxl: { 5764967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get(emitBinaryBuiltin(*this, E, Intrinsic::maxnum)); 5774967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 578ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian case Builtin::BI__builtin_conj: 579ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian case Builtin::BI__builtin_conjf: 580ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian case Builtin::BI__builtin_conjl: { 581ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0)); 582ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian Value *Real = ComplexVal.first; 583ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian Value *Imag = ComplexVal.second; 584258f930227c1a102c9c22eee88df65f748863425Jim Grosbach Value *Zero = 585258f930227c1a102c9c22eee88df65f748863425Jim Grosbach Imag->getType()->isFPOrFPVectorTy() 586ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian ? llvm::ConstantFP::getZeroValueForNegation(Imag->getType()) 587ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian : llvm::Constant::getNullValue(Imag->getType()); 588258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 589ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian Imag = Builder.CreateFSub(Zero, Imag, "sub"); 590ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian return RValue::getComplex(std::make_pair(Real, Imag)); 591ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian } 592ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian case Builtin::BI__builtin_creal: 593ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian case Builtin::BI__builtin_crealf: 59408cc03fcd97c3886b004d861aadbc28de70092c5Meador Inge case Builtin::BI__builtin_creall: 59508cc03fcd97c3886b004d861aadbc28de70092c5Meador Inge case Builtin::BIcreal: 59608cc03fcd97c3886b004d861aadbc28de70092c5Meador Inge case Builtin::BIcrealf: 59708cc03fcd97c3886b004d861aadbc28de70092c5Meador Inge case Builtin::BIcreall: { 598ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0)); 599ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian return RValue::get(ComplexVal.first); 600ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian } 601258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 602ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian case Builtin::BI__builtin_cimag: 603ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian case Builtin::BI__builtin_cimagf: 60408cc03fcd97c3886b004d861aadbc28de70092c5Meador Inge case Builtin::BI__builtin_cimagl: 60508cc03fcd97c3886b004d861aadbc28de70092c5Meador Inge case Builtin::BIcimag: 60608cc03fcd97c3886b004d861aadbc28de70092c5Meador Inge case Builtin::BIcimagf: 60708cc03fcd97c3886b004d861aadbc28de70092c5Meador Inge case Builtin::BIcimagl: { 608ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0)); 609ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian return RValue::get(ComplexVal.second); 610ce23bb7cdc51db26411d75da3b20818ac2cc1db6Fariborz Jahanian } 611258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 612a49a2839266f0bac2b6286e9f49fdc0c292938feBenjamin Kramer case Builtin::BI__builtin_ctzs: 6133a31d60cffedfb7c9e6d129a5c9ba15fa74f179aAnders Carlsson case Builtin::BI__builtin_ctz: 6143a31d60cffedfb7c9e6d129a5c9ba15fa74f179aAnders Carlsson case Builtin::BI__builtin_ctzl: 6153a31d60cffedfb7c9e6d129a5c9ba15fa74f179aAnders Carlsson case Builtin::BI__builtin_ctzll: { 6163a31d60cffedfb7c9e6d129a5c9ba15fa74f179aAnders Carlsson Value *ArgValue = EmitScalarExpr(E->getArg(0)); 6171eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 6189cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *ArgType = ArgValue->getType(); 6198dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType); 6203a31d60cffedfb7c9e6d129a5c9ba15fa74f179aAnders Carlsson 6212acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ResultType = ConvertType(E->getType()); 62264aa4b3ec7e62288e2e66c1935487ece995ca94bJohn McCall Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef()); 623b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef}); 6243a31d60cffedfb7c9e6d129a5c9ba15fa74f179aAnders Carlsson if (Result->getType() != ResultType) 625eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 626eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands "cast"); 6273a31d60cffedfb7c9e6d129a5c9ba15fa74f179aAnders Carlsson return RValue::get(Result); 6283a31d60cffedfb7c9e6d129a5c9ba15fa74f179aAnders Carlsson } 629a49a2839266f0bac2b6286e9f49fdc0c292938feBenjamin Kramer case Builtin::BI__builtin_clzs: 630f4e853340590d5c32e58379e8c379ea1777d3101Eli Friedman case Builtin::BI__builtin_clz: 631f4e853340590d5c32e58379e8c379ea1777d3101Eli Friedman case Builtin::BI__builtin_clzl: 632f4e853340590d5c32e58379e8c379ea1777d3101Eli Friedman case Builtin::BI__builtin_clzll: { 633f4e853340590d5c32e58379e8c379ea1777d3101Eli Friedman Value *ArgValue = EmitScalarExpr(E->getArg(0)); 6341eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 6359cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *ArgType = ArgValue->getType(); 6368dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType); 637f4e853340590d5c32e58379e8c379ea1777d3101Eli Friedman 6382acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ResultType = ConvertType(E->getType()); 63964aa4b3ec7e62288e2e66c1935487ece995ca94bJohn McCall Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef()); 640b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef}); 641f4e853340590d5c32e58379e8c379ea1777d3101Eli Friedman if (Result->getType() != ResultType) 642eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 643eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands "cast"); 644f4e853340590d5c32e58379e8c379ea1777d3101Eli Friedman return RValue::get(Result); 645f4e853340590d5c32e58379e8c379ea1777d3101Eli Friedman } 64604b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_ffs: 64704b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_ffsl: 64804b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_ffsll: { 64904b290030eee33295600728450f348989d1a627eDaniel Dunbar // ffs(x) -> x ? cttz(x) + 1 : 0 65004b290030eee33295600728450f348989d1a627eDaniel Dunbar Value *ArgValue = EmitScalarExpr(E->getArg(0)); 6511eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 6529cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *ArgType = ArgValue->getType(); 6538dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType); 6541eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 6552acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ResultType = ConvertType(E->getType()); 656b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *Tmp = 657b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Builder.CreateAdd(Builder.CreateCall(F, {ArgValue, Builder.getTrue()}), 658b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::ConstantInt::get(ArgType, 1)); 659c9c88b4159791c48e486ca94e3743b5979e2b7a6Owen Anderson Value *Zero = llvm::Constant::getNullValue(ArgType); 66004b290030eee33295600728450f348989d1a627eDaniel Dunbar Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero"); 66104b290030eee33295600728450f348989d1a627eDaniel Dunbar Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp, "ffs"); 66204b290030eee33295600728450f348989d1a627eDaniel Dunbar if (Result->getType() != ResultType) 663eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 664eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands "cast"); 66504b290030eee33295600728450f348989d1a627eDaniel Dunbar return RValue::get(Result); 66604b290030eee33295600728450f348989d1a627eDaniel Dunbar } 66704b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_parity: 66804b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_parityl: 66904b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_parityll: { 67004b290030eee33295600728450f348989d1a627eDaniel Dunbar // parity(x) -> ctpop(x) & 1 67104b290030eee33295600728450f348989d1a627eDaniel Dunbar Value *ArgValue = EmitScalarExpr(E->getArg(0)); 6721eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 6739cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *ArgType = ArgValue->getType(); 6748dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType); 6751eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 6762acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ResultType = ConvertType(E->getType()); 677578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer Value *Tmp = Builder.CreateCall(F, ArgValue); 678578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1)); 67904b290030eee33295600728450f348989d1a627eDaniel Dunbar if (Result->getType() != ResultType) 680eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 681eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands "cast"); 68204b290030eee33295600728450f348989d1a627eDaniel Dunbar return RValue::get(Result); 68304b290030eee33295600728450f348989d1a627eDaniel Dunbar } 68404b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_popcount: 68504b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_popcountl: 68604b290030eee33295600728450f348989d1a627eDaniel Dunbar case Builtin::BI__builtin_popcountll: { 68704b290030eee33295600728450f348989d1a627eDaniel Dunbar Value *ArgValue = EmitScalarExpr(E->getArg(0)); 6881eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 6899cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *ArgType = ArgValue->getType(); 6908dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType); 6911eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 6922acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *ResultType = ConvertType(E->getType()); 693578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer Value *Result = Builder.CreateCall(F, ArgValue); 69404b290030eee33295600728450f348989d1a627eDaniel Dunbar if (Result->getType() != ResultType) 695eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 696eac73e5b3eb3862945bcaa2770c71a727a3ee542Duncan Sands "cast"); 69704b290030eee33295600728450f348989d1a627eDaniel Dunbar return RValue::get(Result); 69804b290030eee33295600728450f348989d1a627eDaniel Dunbar } 69987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case Builtin::BI__builtin_unpredictable: { 70087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Always return the argument of __builtin_unpredictable. LLVM does not 70187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // handle this builtin. Metadata for this builtin should be added directly 70287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // to instructions such as branches or switches that use it. 70387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return RValue::get(EmitScalarExpr(E->getArg(0))); 70487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 705e42b8a596886fc98e367c73e54d761446700029eFariborz Jahanian case Builtin::BI__builtin_expect: { 706dd697bc8629f0fa6777245610d52ca7ceb3b67f4Fariborz Jahanian Value *ArgValue = EmitScalarExpr(E->getArg(0)); 7079cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *ArgType = ArgValue->getType(); 708558229f267d2cdee69639df04382082e7c7ad54cJakub Staszak 709558229f267d2cdee69639df04382082e7c7ad54cJakub Staszak Value *ExpectedValue = EmitScalarExpr(E->getArg(1)); 7100e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Don't generate llvm.expect on -O0 as the backend won't use it for 7110e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // anything. 7120e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Note, we still IRGen ExpectedValue because it could have side-effects. 7130e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (CGM.getCodeGenOpts().OptimizationLevel == 0) 7140e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return RValue::get(ArgValue); 715558229f267d2cdee69639df04382082e7c7ad54cJakub Staszak 7160e2c34f92f00628d48968dfea096d36381f494cbStephen Hines Value *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType); 717b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *Result = 718b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue}, "expval"); 719558229f267d2cdee69639df04382082e7c7ad54cJakub Staszak return RValue::get(Result); 720e42b8a596886fc98e367c73e54d761446700029eFariborz Jahanian } 721176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__builtin_assume_aligned: { 722176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *PtrValue = EmitScalarExpr(E->getArg(0)); 723176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *OffsetValue = 724176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) : nullptr; 725176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 726176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *AlignmentValue = EmitScalarExpr(E->getArg(1)); 727176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue); 728176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines unsigned Alignment = (unsigned) AlignmentCI->getZExtValue(); 729176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 730176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines EmitAlignmentAssumption(PtrValue, Alignment, OffsetValue); 731176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return RValue::get(PtrValue); 732176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 733176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__assume: 734176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__builtin_assume: { 735176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (E->getArg(0)->HasSideEffects(getContext())) 736176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return RValue::get(nullptr); 737176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 738176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *ArgValue = EmitScalarExpr(E->getArg(0)); 739176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *FnAssume = CGM.getIntrinsic(Intrinsic::assume); 740176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return RValue::get(Builder.CreateCall(FnAssume, ArgValue)); 741176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 742d190057934331390ff67ebf51d66186dd5e392f0Benjamin Kramer case Builtin::BI__builtin_bswap16: 743df4852ac816e6050d53b808b86d7c1c9738eb99eAnders Carlsson case Builtin::BI__builtin_bswap32: 744df4852ac816e6050d53b808b86d7c1c9738eb99eAnders Carlsson case Builtin::BI__builtin_bswap64: { 7454967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bswap)); 7464967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 7474967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_bitreverse8: 7484967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_bitreverse16: 7494967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_bitreverse32: 7504967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_bitreverse64: { 7514967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bitreverse)); 7521eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } 753d5f8a4fd4d6dfb0415b93bb7ab721bba5cab1332Daniel Dunbar case Builtin::BI__builtin_object_size: { 75487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar unsigned Type = 75587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar E->getArg(1)->EvaluateKnownConstInt(getContext()).getZExtValue(); 75687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar auto *ResType = cast<llvm::IntegerType>(ConvertType(E->getType())); 75787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 75887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // We pass this builtin onto the optimizer so that it can figure out the 75987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // object size in more complex cases. 76087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return RValue::get(emitBuiltinObjectSize(E->getArg(0), Type, ResType)); 761d5f8a4fd4d6dfb0415b93bb7ab721bba5cab1332Daniel Dunbar } 7624493f79fce48cd9cbd9f55fa9d452cde736747a0Daniel Dunbar case Builtin::BI__builtin_prefetch: { 7634493f79fce48cd9cbd9f55fa9d452cde736747a0Daniel Dunbar Value *Locality, *RW, *Address = EmitScalarExpr(E->getArg(0)); 7644493f79fce48cd9cbd9f55fa9d452cde736747a0Daniel Dunbar // FIXME: Technically these constants should of type 'int', yes? 7651eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump RW = (E->getNumArgs() > 1) ? EmitScalarExpr(E->getArg(1)) : 76677b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner llvm::ConstantInt::get(Int32Ty, 0); 7671eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump Locality = (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) : 76877b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner llvm::ConstantInt::get(Int32Ty, 3); 7692eccb672799d19fb535ce349566fea6729cbb891Bruno Cardoso Lopes Value *Data = llvm::ConstantInt::get(Int32Ty, 1); 7708dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::prefetch); 771b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return RValue::get(Builder.CreateCall(F, {Address, RW, Locality, Data})); 7724493f79fce48cd9cbd9f55fa9d452cde736747a0Daniel Dunbar } 773a841c19f7860393d6319bf40e9d662284462771dHal Finkel case Builtin::BI__builtin_readcyclecounter: { 774a841c19f7860393d6319bf40e9d662284462771dHal Finkel Value *F = CGM.getIntrinsic(Intrinsic::readcyclecounter); 77587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return RValue::get(Builder.CreateCall(F)); 776a841c19f7860393d6319bf40e9d662284462771dHal Finkel } 777651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case Builtin::BI__builtin___clear_cache: { 778651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Begin = EmitScalarExpr(E->getArg(0)); 779651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *End = EmitScalarExpr(E->getArg(1)); 780651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *F = CGM.getIntrinsic(Intrinsic::clear_cache); 781b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return RValue::get(Builder.CreateCall(F, {Begin, End})); 782651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 78387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case Builtin::BI__builtin_trap: 78487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return RValue::get(EmitTrapCall(Intrinsic::trap)); 78587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case Builtin::BI__debugbreak: 78687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return RValue::get(EmitTrapCall(Intrinsic::debugtrap)); 78721190d54634d6e244e85d28ad915ce2fe86ecbffChris Lattner case Builtin::BI__builtin_unreachable: { 788176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (SanOpts.has(SanitizerKind::Unreachable)) { 789176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines SanitizerScope SanScope(this); 790176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines EmitCheck(std::make_pair(static_cast<llvm::Value *>(Builder.getFalse()), 791176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines SanitizerKind::Unreachable), 792176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines "builtin_unreachable", EmitCheckSourceLocation(E->getExprLoc()), 793176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines None); 794176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } else 795cd5b22e12b6513163dd131589746c194090f14e6John McCall Builder.CreateUnreachable(); 796cd5b22e12b6513163dd131589746c194090f14e6John McCall 797cd5b22e12b6513163dd131589746c194090f14e6John McCall // We do need to preserve an insertion point. 798d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall EmitBlock(createBasicBlock("unreachable.cont")); 799cd5b22e12b6513163dd131589746c194090f14e6John McCall 8006bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 80121190d54634d6e244e85d28ad915ce2fe86ecbffChris Lattner } 802258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 803a933c3c052bbd87b01cc6fc7a7745e1c4b1757fbDaniel Dunbar case Builtin::BI__builtin_powi: 804a933c3c052bbd87b01cc6fc7a7745e1c4b1757fbDaniel Dunbar case Builtin::BI__builtin_powif: 805a933c3c052bbd87b01cc6fc7a7745e1c4b1757fbDaniel Dunbar case Builtin::BI__builtin_powil: { 806a933c3c052bbd87b01cc6fc7a7745e1c4b1757fbDaniel Dunbar Value *Base = EmitScalarExpr(E->getArg(0)); 807a933c3c052bbd87b01cc6fc7a7745e1c4b1757fbDaniel Dunbar Value *Exponent = EmitScalarExpr(E->getArg(1)); 8089cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *ArgType = Base->getType(); 8098dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::powi, ArgType); 810b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return RValue::get(Builder.CreateCall(F, {Base, Exponent})); 811a933c3c052bbd87b01cc6fc7a7745e1c4b1757fbDaniel Dunbar } 812a933c3c052bbd87b01cc6fc7a7745e1c4b1757fbDaniel Dunbar 813fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_isgreater: 814fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_isgreaterequal: 815fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_isless: 816fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_islessequal: 817fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_islessgreater: 818fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_isunordered: { 819fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner // Ordered comparisons: we know the arguments to these are matching scalar 820fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner // floating point values. 8211eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump Value *LHS = EmitScalarExpr(E->getArg(0)); 822fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner Value *RHS = EmitScalarExpr(E->getArg(1)); 8231eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 824fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner switch (BuiltinID) { 825b219cfc4d75f0a03630b7c4509ef791b7e97b2c8David Blaikie default: llvm_unreachable("Unknown ordered comparison"); 826fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_isgreater: 827fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner LHS = Builder.CreateFCmpOGT(LHS, RHS, "cmp"); 828fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner break; 829fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_isgreaterequal: 830fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner LHS = Builder.CreateFCmpOGE(LHS, RHS, "cmp"); 831fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner break; 832fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_isless: 833fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner LHS = Builder.CreateFCmpOLT(LHS, RHS, "cmp"); 834fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner break; 835fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_islessequal: 836fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner LHS = Builder.CreateFCmpOLE(LHS, RHS, "cmp"); 837fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner break; 838fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner case Builtin::BI__builtin_islessgreater: 839fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner LHS = Builder.CreateFCmpONE(LHS, RHS, "cmp"); 840fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner break; 8411eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump case Builtin::BI__builtin_isunordered: 842fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner LHS = Builder.CreateFCmpUNO(LHS, RHS, "cmp"); 843fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner break; 844fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner } 845fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner // ZExt bool to int type. 846578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer return RValue::get(Builder.CreateZExt(LHS, ConvertType(E->getType()))); 847fe23e217774aaec350086fab839210d7d9e1e3f4Chris Lattner } 848d6139895f43d161a972d134ffda4229d2f548eb6Eli Friedman case Builtin::BI__builtin_isnan: { 849d6139895f43d161a972d134ffda4229d2f548eb6Eli Friedman Value *V = EmitScalarExpr(E->getArg(0)); 850d6139895f43d161a972d134ffda4229d2f548eb6Eli Friedman V = Builder.CreateFCmpUNO(V, V, "cmp"); 851578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); 852d6139895f43d161a972d134ffda4229d2f548eb6Eli Friedman } 853258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 8544967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_isinf: 8554967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_isfinite: { 8564967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // isinf(x) --> fabs(x) == infinity 8574967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // isfinite(x) --> fabs(x) != infinity 8584967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // x != NaN via the ordered compare in either case. 859420b11850d3f4557421f43f519b59d528329c668Chris Lattner Value *V = EmitScalarExpr(E->getArg(0)); 8604967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Fabs = EmitFAbs(*this, V); 8614967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Constant *Infinity = ConstantFP::getInfinity(V->getType()); 8624967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar CmpInst::Predicate Pred = (BuiltinID == Builtin::BI__builtin_isinf) 8634967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar ? CmpInst::FCMP_OEQ 8644967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar : CmpInst::FCMP_ONE; 8654967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *FCmp = Builder.CreateFCmp(Pred, Fabs, Infinity, "cmpinf"); 8664967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get(Builder.CreateZExt(FCmp, ConvertType(E->getType()))); 867420b11850d3f4557421f43f519b59d528329c668Chris Lattner } 868258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 8693ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar case Builtin::BI__builtin_isinf_sign: { 8703ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // isinf_sign(x) -> fabs(x) == infinity ? (signbit(x) ? -1 : 1) : 0 8713ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Value *Arg = EmitScalarExpr(E->getArg(0)); 8723ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Value *AbsArg = EmitFAbs(*this, Arg); 8733ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Value *IsInf = Builder.CreateFCmpOEQ( 8743ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar AbsArg, ConstantFP::getInfinity(Arg->getType()), "isinf"); 8753ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Value *IsNeg = EmitSignBit(*this, Arg); 8763ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 8773ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar llvm::Type *IntTy = ConvertType(E->getType()); 8783ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Value *Zero = Constant::getNullValue(IntTy); 8793ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Value *One = ConstantInt::get(IntTy, 1); 8803ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Value *NegativeOne = ConstantInt::get(IntTy, -1); 8813ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Value *SignResult = Builder.CreateSelect(IsNeg, NegativeOne, One); 8823ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Value *Result = Builder.CreateSelect(IsInf, SignResult, Zero); 8833ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar return RValue::get(Result); 8843ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar } 8856349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer 8866349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer case Builtin::BI__builtin_isnormal: { 8876349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer // isnormal(x) --> x == x && fabsf(x) < infinity && fabsf(x) >= float_min 8886349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer Value *V = EmitScalarExpr(E->getArg(0)); 8896349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq"); 8906349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer 891176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *Abs = EmitFAbs(*this, V); 8926349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer Value *IsLessThanInf = 8936349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer Builder.CreateFCmpULT(Abs, ConstantFP::getInfinity(V->getType()),"isinf"); 8946349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer APFloat Smallest = APFloat::getSmallestNormalized( 8956349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer getContext().getFloatTypeSemantics(E->getArg(0)->getType())); 8966349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer Value *IsNormal = 8976349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer Builder.CreateFCmpUGE(Abs, ConstantFP::get(V->getContext(), Smallest), 8986349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer "isnormal"); 8996349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer V = Builder.CreateAnd(Eq, IsLessThanInf, "and"); 9006349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer V = Builder.CreateAnd(V, IsNormal, "and"); 9016349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); 9026349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer } 9036349ce94d1b4fa560bf060c5ca5ad5728ce4fad9Benjamin Kramer 9047867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer case Builtin::BI__builtin_fpclassify: { 9057867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *V = EmitScalarExpr(E->getArg(5)); 9062acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *Ty = ConvertType(E->getArg(5)->getType()); 9077867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer 9087867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer // Create Result 9097867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer BasicBlock *Begin = Builder.GetInsertBlock(); 9107867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer BasicBlock *End = createBasicBlock("fpclassify_end", this->CurFn); 9117867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.SetInsertPoint(End); 9127867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer PHINode *Result = 913bbf3bacb3e0c1ebb3e8a4a8b1330404a7e379315Jay Foad Builder.CreatePHI(ConvertType(E->getArg(0)->getType()), 4, 9147867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer "fpclassify_result"); 9157867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer 9167867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer // if (V==0) return FP_ZERO 9177867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.SetInsertPoint(Begin); 9187867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *IsZero = Builder.CreateFCmpOEQ(V, Constant::getNullValue(Ty), 9197867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer "iszero"); 9207867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *ZeroLiteral = EmitScalarExpr(E->getArg(4)); 9217867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer BasicBlock *NotZero = createBasicBlock("fpclassify_not_zero", this->CurFn); 9227867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.CreateCondBr(IsZero, End, NotZero); 9237867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Result->addIncoming(ZeroLiteral, Begin); 9247867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer 9257867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer // if (V != V) return FP_NAN 9267867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.SetInsertPoint(NotZero); 9277867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *IsNan = Builder.CreateFCmpUNO(V, V, "cmp"); 9287867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *NanLiteral = EmitScalarExpr(E->getArg(0)); 9297867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer BasicBlock *NotNan = createBasicBlock("fpclassify_not_nan", this->CurFn); 9307867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.CreateCondBr(IsNan, End, NotNan); 9317867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Result->addIncoming(NanLiteral, NotZero); 9327867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer 9337867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer // if (fabs(V) == infinity) return FP_INFINITY 9347867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.SetInsertPoint(NotNan); 935176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *VAbs = EmitFAbs(*this, V); 9367867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *IsInf = 9377867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(V->getType()), 9387867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer "isinf"); 9397867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *InfLiteral = EmitScalarExpr(E->getArg(1)); 9407867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer BasicBlock *NotInf = createBasicBlock("fpclassify_not_inf", this->CurFn); 9417867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.CreateCondBr(IsInf, End, NotInf); 9427867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Result->addIncoming(InfLiteral, NotNan); 9437867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer 9447867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer // if (fabs(V) >= MIN_NORMAL) return FP_NORMAL else FP_SUBNORMAL 9457867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.SetInsertPoint(NotInf); 9467867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer APFloat Smallest = APFloat::getSmallestNormalized( 9477867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer getContext().getFloatTypeSemantics(E->getArg(5)->getType())); 9487867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *IsNormal = 9497867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.CreateFCmpUGE(VAbs, ConstantFP::get(V->getContext(), Smallest), 9507867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer "isnormal"); 9517867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Value *NormalResult = 9527867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.CreateSelect(IsNormal, EmitScalarExpr(E->getArg(2)), 9537867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer EmitScalarExpr(E->getArg(3))); 9547867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.CreateBr(End); 9557867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Result->addIncoming(NormalResult, NotInf); 9567867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer 9577867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer // return Result 9587867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer Builder.SetInsertPoint(End); 9597867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer return RValue::get(Result); 9607867f1a62b8b42cc2a55cc571608a75db2d516e0Benjamin Kramer } 961258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 962b52fe9ce99970955a5f581f5c66fcd89be9a268bEli Friedman case Builtin::BIalloca: 9636526de425dbfd368407566ff123b5bdc0924859aReid Kleckner case Builtin::BI_alloca: 9649e800e3dd80d77f6c47054738177bf824089f55aChris Lattner case Builtin::BI__builtin_alloca: { 9659e800e3dd80d77f6c47054738177bf824089f55aChris Lattner Value *Size = EmitScalarExpr(E->getArg(0)); 966578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer return RValue::get(Builder.CreateAlloca(Builder.getInt8Ty(), Size)); 9671caae959017b355e9bb61250d5a0d04edbf468b0Daniel Dunbar } 968e6dddfd907f6ea58daed5e26eeaacd893d98db9bEli Friedman case Builtin::BIbzero: 9691caae959017b355e9bb61250d5a0d04edbf468b0Daniel Dunbar case Builtin::BI__builtin_bzero: { 97087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Dest = EmitPointerWithAlignment(E->getArg(0)); 9713ecd785aff34381f3704d9cb28fe3ef85af759deMon P Wang Value *SizeVal = EmitScalarExpr(E->getArg(1)); 97287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), 973b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar E->getArg(0)->getExprLoc(), FD, 0); 97487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal, false); 97587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return RValue::get(Dest.getPointer()); 9769e800e3dd80d77f6c47054738177bf824089f55aChris Lattner } 977e6ec205d6d0f4aec27bf49ca1e8fbb139acc2f2bEli Friedman case Builtin::BImemcpy: 978d4b32e46517358f34e8cfbea35010adfcc3786e0Eli Friedman case Builtin::BI__builtin_memcpy: { 97987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Dest = EmitPointerWithAlignment(E->getArg(0)); 98087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Src = EmitPointerWithAlignment(E->getArg(1)); 9813ecd785aff34381f3704d9cb28fe3ef85af759deMon P Wang Value *SizeVal = EmitScalarExpr(E->getArg(2)); 98287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), 983b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar E->getArg(0)->getExprLoc(), FD, 0); 98487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(), 985b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar E->getArg(1)->getExprLoc(), FD, 1); 98687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateMemCpy(Dest, Src, SizeVal, false); 98787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return RValue::get(Dest.getPointer()); 9881caae959017b355e9bb61250d5a0d04edbf468b0Daniel Dunbar } 989258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 990a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner case Builtin::BI__builtin___memcpy_chk: { 991f3477c13eeaf11b32a41f181398fb5deffd0dd73Sylvestre Ledru // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memcpy iff cst1<=cst2. 992a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith llvm::APSInt Size, DstSize; 993a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) || 994a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext())) 995a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner break; 996a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner if (Size.ugt(DstSize)) 997a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner break; 99887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Dest = EmitPointerWithAlignment(E->getArg(0)); 99987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Src = EmitPointerWithAlignment(E->getArg(1)); 1000a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); 100187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateMemCpy(Dest, Src, SizeVal, false); 100287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return RValue::get(Dest.getPointer()); 1003a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner } 1004258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 10058e2eab27056a78bf1db50ee09929438ed5ea9d93Fariborz Jahanian case Builtin::BI__builtin_objc_memmove_collectable: { 100687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address DestAddr = EmitPointerWithAlignment(E->getArg(0)); 100787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address SrcAddr = EmitPointerWithAlignment(E->getArg(1)); 100855bcace250e1ff366e4482714b344b8cbc8be5f3Fariborz Jahanian Value *SizeVal = EmitScalarExpr(E->getArg(2)); 1009258f930227c1a102c9c22eee88df65f748863425Jim Grosbach CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, 101087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar DestAddr, SrcAddr, SizeVal); 101187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return RValue::get(DestAddr.getPointer()); 101255bcace250e1ff366e4482714b344b8cbc8be5f3Fariborz Jahanian } 1013a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner 1014a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner case Builtin::BI__builtin___memmove_chk: { 1015f3477c13eeaf11b32a41f181398fb5deffd0dd73Sylvestre Ledru // fold __builtin_memmove_chk(x, y, cst1, cst2) to memmove iff cst1<=cst2. 1016a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith llvm::APSInt Size, DstSize; 1017a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) || 1018a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext())) 1019a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner break; 1020a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner if (Size.ugt(DstSize)) 1021a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner break; 102287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Dest = EmitPointerWithAlignment(E->getArg(0)); 102387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Src = EmitPointerWithAlignment(E->getArg(1)); 1024a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); 102587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateMemMove(Dest, Src, SizeVal, false); 102687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return RValue::get(Dest.getPointer()); 1027a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner } 1028a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner 1029e6ec205d6d0f4aec27bf49ca1e8fbb139acc2f2bEli Friedman case Builtin::BImemmove: 10301caae959017b355e9bb61250d5a0d04edbf468b0Daniel Dunbar case Builtin::BI__builtin_memmove: { 103187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Dest = EmitPointerWithAlignment(E->getArg(0)); 103287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Src = EmitPointerWithAlignment(E->getArg(1)); 10333ecd785aff34381f3704d9cb28fe3ef85af759deMon P Wang Value *SizeVal = EmitScalarExpr(E->getArg(2)); 103487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), 1035b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar E->getArg(0)->getExprLoc(), FD, 0); 103687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(), 1037b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar E->getArg(1)->getExprLoc(), FD, 1); 103887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateMemMove(Dest, Src, SizeVal, false); 103987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return RValue::get(Dest.getPointer()); 10401caae959017b355e9bb61250d5a0d04edbf468b0Daniel Dunbar } 1041e6ec205d6d0f4aec27bf49ca1e8fbb139acc2f2bEli Friedman case Builtin::BImemset: 10421caae959017b355e9bb61250d5a0d04edbf468b0Daniel Dunbar case Builtin::BI__builtin_memset: { 104387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Dest = EmitPointerWithAlignment(E->getArg(0)); 10449f0c7cc36d29cf591c33962931f5862847145f3eBenjamin Kramer Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), 10459f0c7cc36d29cf591c33962931f5862847145f3eBenjamin Kramer Builder.getInt8Ty()); 10463ecd785aff34381f3704d9cb28fe3ef85af759deMon P Wang Value *SizeVal = EmitScalarExpr(E->getArg(2)); 104787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), 1048b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar E->getArg(0)->getExprLoc(), FD, 0); 104987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateMemSet(Dest, ByteVal, SizeVal, false); 105087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return RValue::get(Dest.getPointer()); 1051d4b32e46517358f34e8cfbea35010adfcc3786e0Eli Friedman } 1052a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner case Builtin::BI__builtin___memset_chk: { 1053f3477c13eeaf11b32a41f181398fb5deffd0dd73Sylvestre Ledru // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2. 1054a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith llvm::APSInt Size, DstSize; 1055a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) || 1056a6b8b2c09610b8bc4330e948ece8b940c2386406Richard Smith !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext())) 1057a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner break; 1058a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner if (Size.ugt(DstSize)) 1059a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner break; 106087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Dest = EmitPointerWithAlignment(E->getArg(0)); 1061a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), 1062a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner Builder.getInt8Ty()); 1063a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); 106487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateMemSet(Dest, ByteVal, SizeVal, false); 106587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return RValue::get(Dest.getPointer()); 1066a5e5e0f41e1dcee4603244ccea3d3956c55c23acChris Lattner } 1067fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall case Builtin::BI__builtin_dwarf_cfa: { 1068fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall // The offset in bytes from the first argument to the CFA. 1069fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall // 1070fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall // Why on earth is this in the frontend? Is there any reason at 1071fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall // all that the backend can't reasonably determine this while 1072fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall // lowering llvm.eh.dwarf.cfa()? 1073fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall // 1074fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall // TODO: If there's a satisfactory reason, add a target hook for 1075fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall // this instead of hard-coding 0, which is correct for most targets. 1076fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall int32_t Offset = 0; 1077fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall 10788dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa); 1079258f930227c1a102c9c22eee88df65f748863425Jim Grosbach return RValue::get(Builder.CreateCall(F, 108077b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner llvm::ConstantInt::get(Int32Ty, Offset))); 1081fb17a562135dd7597121da9245d0c1bdcda4146fJohn McCall } 1082256f77e431bc6b920ec94cf0bb4ad339ca21b8c9Eli Friedman case Builtin::BI__builtin_return_address: { 108387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Depth = 108487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar CGM.EmitConstantExpr(E->getArg(0), getContext().UnsignedIntTy, this); 10858dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::returnaddress); 108683c2a98012a65b51be66fd76c3a1b13ed782c558Anton Korobeynikov return RValue::get(Builder.CreateCall(F, Depth)); 1087256f77e431bc6b920ec94cf0bb4ad339ca21b8c9Eli Friedman } 1088256f77e431bc6b920ec94cf0bb4ad339ca21b8c9Eli Friedman case Builtin::BI__builtin_frame_address: { 108987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Depth = 109087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar CGM.EmitConstantExpr(E->getArg(0), getContext().UnsignedIntTy, this); 10918dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::frameaddress); 109283c2a98012a65b51be66fd76c3a1b13ed782c558Anton Korobeynikov return RValue::get(Builder.CreateCall(F, Depth)); 1093256f77e431bc6b920ec94cf0bb4ad339ca21b8c9Eli Friedman } 10943b660efb9f9fa3e87096f4a96a2093cd17c43c2eEli Friedman case Builtin::BI__builtin_extract_return_addr: { 1095492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall Value *Address = EmitScalarExpr(E->getArg(0)); 1096492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall Value *Result = getTargetHooks().decodeReturnAddress(*this, Address); 1097492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall return RValue::get(Result); 1098492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall } 1099492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall case Builtin::BI__builtin_frob_return_addr: { 1100492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall Value *Address = EmitScalarExpr(E->getArg(0)); 1101492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall Value *Result = getTargetHooks().encodeReturnAddress(*this, Address); 1102492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall return RValue::get(Result); 11033b660efb9f9fa3e87096f4a96a2093cd17c43c2eEli Friedman } 11046374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall case Builtin::BI__builtin_dwarf_sp_column: { 11052acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::IntegerType *Ty 11066374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall = cast<llvm::IntegerType>(ConvertType(E->getType())); 11076374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall int Column = getTargetHooks().getDwarfEHStackPointer(CGM); 11086374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall if (Column == -1) { 11096374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall CGM.ErrorUnsupported(E, "__builtin_dwarf_sp_column"); 11106374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall return RValue::get(llvm::UndefValue::get(Ty)); 11116374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall } 11126374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall return RValue::get(llvm::ConstantInt::get(Ty, Column, true)); 11136374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall } 11146374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall case Builtin::BI__builtin_init_dwarf_reg_size_table: { 11156374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall Value *Address = EmitScalarExpr(E->getArg(0)); 11166374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall if (getTargetHooks().initDwarfEHRegSizeTable(*this, Address)) 11176374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall CGM.ErrorUnsupported(E, "__builtin_init_dwarf_reg_size_table"); 11186374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall return RValue::get(llvm::UndefValue::get(ConvertType(E->getType()))); 11196374c3307e2d73348f7b8cc73eeeb0998ad0ac94John McCall } 11207ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall case Builtin::BI__builtin_eh_return: { 11217ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall Value *Int = EmitScalarExpr(E->getArg(0)); 11227ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall Value *Ptr = EmitScalarExpr(E->getArg(1)); 11237ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall 11242acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType()); 11257ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) && 11267ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall "LLVM's __builtin_eh_return only supports 32- and 64-bit variants"); 11277ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall Value *F = CGM.getIntrinsic(IntTy->getBitWidth() == 32 11287ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall ? Intrinsic::eh_return_i32 11298dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer : Intrinsic::eh_return_i64); 1130b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Builder.CreateCall(F, {Int, Ptr}); 1131cd5b22e12b6513163dd131589746c194090f14e6John McCall Builder.CreateUnreachable(); 1132cd5b22e12b6513163dd131589746c194090f14e6John McCall 1133cd5b22e12b6513163dd131589746c194090f14e6John McCall // We do need to preserve an insertion point. 1134d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall EmitBlock(createBasicBlock("builtin_eh_return.cont")); 1135cd5b22e12b6513163dd131589746c194090f14e6John McCall 11366bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 11377ada111fd5e81aff355e67bad0e4083f552b34bdJohn McCall } 1138a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman case Builtin::BI__builtin_unwind_init: { 11398dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init); 114087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return RValue::get(Builder.CreateCall(F)); 1141a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman } 11425e11085830d4d4c53ff75575ab75889ee5126854John McCall case Builtin::BI__builtin_extend_pointer: { 11435e11085830d4d4c53ff75575ab75889ee5126854John McCall // Extends a pointer to the size of an _Unwind_Word, which is 1144d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // uint64_t on all platforms. Generally this gets poked into a 1145d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // register and eventually used as an address, so if the 1146d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // addressing registers are wider than pointers and the platform 1147d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // doesn't implicitly ignore high-order bits when doing 1148d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // addressing, we need to make sure we zext / sext based on 1149d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // the platform's expectations. 11505e11085830d4d4c53ff75575ab75889ee5126854John McCall // 11515e11085830d4d4c53ff75575ab75889ee5126854John McCall // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html 1152d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall 1153d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // Cast the pointer to intptr_t. 11545e11085830d4d4c53ff75575ab75889ee5126854John McCall Value *Ptr = EmitScalarExpr(E->getArg(0)); 1155d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall Value *Result = Builder.CreatePtrToInt(Ptr, IntPtrTy, "extend.cast"); 1156d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall 1157d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // If that's 64 bits, we're done. 1158d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall if (IntPtrTy->getBitWidth() == 64) 1159d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall return RValue::get(Result); 1160d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall 1161d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall // Otherwise, ask the codegen data what to do. 1162492c4f998d848673d3d6c9e6416115df4036a71dJohn McCall if (getTargetHooks().extendPointerWithSExt()) 1163d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall return RValue::get(Builder.CreateSExt(Result, Int64Ty, "extend.sext")); 1164d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall else 1165d0b76ca10feefcfda5cb16698e50197e87a7d876John McCall return RValue::get(Builder.CreateZExt(Result, Int64Ty, "extend.zext")); 11665e11085830d4d4c53ff75575ab75889ee5126854John McCall } 1167a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman case Builtin::BI__builtin_setjmp: { 116878673d9f910e8dfe13248c2426c51d8f9fb28572John McCall // Buffer is a void**. 116987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Buf = EmitPointerWithAlignment(E->getArg(0)); 117078673d9f910e8dfe13248c2426c51d8f9fb28572John McCall 117178673d9f910e8dfe13248c2426c51d8f9fb28572John McCall // Store the frame pointer to the setjmp buffer. 1172a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman Value *FrameAddr = 117378673d9f910e8dfe13248c2426c51d8f9fb28572John McCall Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress), 117477b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner ConstantInt::get(Int32Ty, 0)); 1175a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman Builder.CreateStore(FrameAddr, Buf); 117678673d9f910e8dfe13248c2426c51d8f9fb28572John McCall 11776d172e2985346e55095c75f456901ea5d40fddaaJim Grosbach // Store the stack pointer to the setjmp buffer. 11786d172e2985346e55095c75f456901ea5d40fddaaJim Grosbach Value *StackAddr = 117987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave)); 118087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address StackSaveSlot = 118187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateConstInBoundsGEP(Buf, 2, getPointerSize()); 11826d172e2985346e55095c75f456901ea5d40fddaaJim Grosbach Builder.CreateStore(StackAddr, StackSaveSlot); 11836d172e2985346e55095c75f456901ea5d40fddaaJim Grosbach 118478673d9f910e8dfe13248c2426c51d8f9fb28572John McCall // Call LLVM's EH setjmp, which is lightweight. 118578673d9f910e8dfe13248c2426c51d8f9fb28572John McCall Value *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp); 1186d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Buf = Builder.CreateBitCast(Buf, Int8PtrTy); 118787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return RValue::get(Builder.CreateCall(F, Buf.getPointer())); 1188a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman } 1189a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman case Builtin::BI__builtin_longjmp: { 1190a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman Value *Buf = EmitScalarExpr(E->getArg(0)); 1191d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Buf = Builder.CreateBitCast(Buf, Int8PtrTy); 119278673d9f910e8dfe13248c2426c51d8f9fb28572John McCall 119378673d9f910e8dfe13248c2426c51d8f9fb28572John McCall // Call LLVM's EH longjmp, which is lightweight. 119478673d9f910e8dfe13248c2426c51d8f9fb28572John McCall Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf); 119578673d9f910e8dfe13248c2426c51d8f9fb28572John McCall 1196cd5b22e12b6513163dd131589746c194090f14e6John McCall // longjmp doesn't return; mark this as unreachable. 1197cd5b22e12b6513163dd131589746c194090f14e6John McCall Builder.CreateUnreachable(); 1198cd5b22e12b6513163dd131589746c194090f14e6John McCall 1199cd5b22e12b6513163dd131589746c194090f14e6John McCall // We do need to preserve an insertion point. 1200d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall EmitBlock(createBasicBlock("longjmp.cont")); 1201cd5b22e12b6513163dd131589746c194090f14e6John McCall 12026bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 1203a6d75c0324ac690107bbaa7193b526ef21466212Eli Friedman } 12041ffe281890f3cd7728316b45a1f3dd4d3120af7bMon P Wang case Builtin::BI__sync_fetch_and_add: 12051ffe281890f3cd7728316b45a1f3dd4d3120af7bMon P Wang case Builtin::BI__sync_fetch_and_sub: 12065caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_or: 12075caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_and: 12085caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_xor: 1209176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__sync_fetch_and_nand: 12105caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_add_and_fetch: 12115caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_sub_and_fetch: 12125caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_and_and_fetch: 12135caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_or_and_fetch: 12145caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_xor_and_fetch: 1215176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__sync_nand_and_fetch: 12165caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_val_compare_and_swap: 12175caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_bool_compare_and_swap: 12185caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_test_and_set: 12195caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_release: 122023aa9c8ca0bc441aab2a38a4c9fc7a1c9e9ac16aChris Lattner case Builtin::BI__sync_swap: 1221b219cfc4d75f0a03630b7c4509ef791b7e97b2c8David Blaikie llvm_unreachable("Shouldn't make it through sema"); 12225caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_add_1: 12235caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_add_2: 12245caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_add_4: 12255caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_add_8: 12265caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_add_16: 1227c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Add, E); 12285caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_sub_1: 12295caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_sub_2: 12305caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_sub_4: 12315caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_sub_8: 12325caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_sub_16: 1233c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Sub, E); 12345caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_or_1: 12355caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_or_2: 12365caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_or_4: 12375caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_or_8: 12385caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_or_16: 1239c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Or, E); 12405caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_and_1: 12415caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_and_2: 12425caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_and_4: 12435caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_and_8: 12445caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_and_16: 1245c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::And, E); 12465caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_xor_1: 12475caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_xor_2: 12485caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_xor_4: 12495caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_xor_8: 12505caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_fetch_and_xor_16: 1251c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xor, E); 1252176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__sync_fetch_and_nand_1: 1253176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__sync_fetch_and_nand_2: 1254176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__sync_fetch_and_nand_4: 1255176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__sync_fetch_and_nand_8: 1256176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__sync_fetch_and_nand_16: 1257176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Nand, E); 12581eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 12595caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner // Clang extensions: not overloaded yet. 12601ffe281890f3cd7728316b45a1f3dd4d3120af7bMon P Wang case Builtin::BI__sync_fetch_and_min: 1261c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Min, E); 12621ffe281890f3cd7728316b45a1f3dd4d3120af7bMon P Wang case Builtin::BI__sync_fetch_and_max: 1263c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Max, E); 12641ffe281890f3cd7728316b45a1f3dd4d3120af7bMon P Wang case Builtin::BI__sync_fetch_and_umin: 1265c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMin, E); 12661ffe281890f3cd7728316b45a1f3dd4d3120af7bMon P Wang case Builtin::BI__sync_fetch_and_umax: 1267c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMax, E); 12680002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar 12695caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_add_and_fetch_1: 12705caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_add_and_fetch_2: 12715caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_add_and_fetch_4: 12725caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_add_and_fetch_8: 12735caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_add_and_fetch_16: 1274c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Add, E, 12750002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar llvm::Instruction::Add); 12765caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_sub_and_fetch_1: 12775caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_sub_and_fetch_2: 12785caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_sub_and_fetch_4: 12795caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_sub_and_fetch_8: 12805caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_sub_and_fetch_16: 1281c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Sub, E, 12820002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar llvm::Instruction::Sub); 12835caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_and_and_fetch_1: 12845caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_and_and_fetch_2: 12855caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_and_and_fetch_4: 12865caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_and_and_fetch_8: 12875caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_and_and_fetch_16: 1288c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::And, E, 12890002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar llvm::Instruction::And); 12905caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_or_and_fetch_1: 12915caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_or_and_fetch_2: 12925caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_or_and_fetch_4: 12935caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_or_and_fetch_8: 12945caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_or_and_fetch_16: 1295c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Or, E, 12960002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar llvm::Instruction::Or); 12975caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_xor_and_fetch_1: 12985caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_xor_and_fetch_2: 12995caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_xor_and_fetch_4: 13005caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_xor_and_fetch_8: 13015caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_xor_and_fetch_16: 1302c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Xor, E, 13030002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar llvm::Instruction::Xor); 1304176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__sync_nand_and_fetch_1: 1305176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__sync_nand_and_fetch_2: 1306176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__sync_nand_and_fetch_4: 1307176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__sync_nand_and_fetch_8: 1308176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__sync_nand_and_fetch_16: 1309176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Nand, E, 1310176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::Instruction::And, true); 13111eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 13125caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_val_compare_and_swap_1: 13135caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_val_compare_and_swap_2: 13145caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_val_compare_and_swap_4: 13155caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_val_compare_and_swap_8: 131687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case Builtin::BI__sync_val_compare_and_swap_16: 131787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return RValue::get(MakeAtomicCmpXchgValue(*this, E, false)); 13180002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar 13195caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_bool_compare_and_swap_1: 13205caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_bool_compare_and_swap_2: 13215caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_bool_compare_and_swap_4: 13225caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_bool_compare_and_swap_8: 132387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case Builtin::BI__sync_bool_compare_and_swap_16: 132487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return RValue::get(MakeAtomicCmpXchgValue(*this, E, true)); 13250002d23aaf10f307273dab5facda01c137283d22Daniel Dunbar 132623aa9c8ca0bc441aab2a38a4c9fc7a1c9e9ac16aChris Lattner case Builtin::BI__sync_swap_1: 132723aa9c8ca0bc441aab2a38a4c9fc7a1c9e9ac16aChris Lattner case Builtin::BI__sync_swap_2: 132823aa9c8ca0bc441aab2a38a4c9fc7a1c9e9ac16aChris Lattner case Builtin::BI__sync_swap_4: 132923aa9c8ca0bc441aab2a38a4c9fc7a1c9e9ac16aChris Lattner case Builtin::BI__sync_swap_8: 133023aa9c8ca0bc441aab2a38a4c9fc7a1c9e9ac16aChris Lattner case Builtin::BI__sync_swap_16: 1331c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E); 133223aa9c8ca0bc441aab2a38a4c9fc7a1c9e9ac16aChris Lattner 13335caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_test_and_set_1: 13345caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_test_and_set_2: 13355caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_test_and_set_4: 13365caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_test_and_set_8: 13375caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_test_and_set_16: 1338c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E); 1339cb61a7bbe635cfa941b1aeaaa1fbda1bf900ee51Daniel Dunbar 13405caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_release_1: 13415caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_release_2: 13425caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_release_4: 13435caa370ea6f70bd3e7e4a9cc3b69ac1a849c8534Chris Lattner case Builtin::BI__sync_lock_release_8: 1344f58cd9bca9c2e7373300fc8bb7c57cff7e4eda4fChris Lattner case Builtin::BI__sync_lock_release_16: { 1345f58cd9bca9c2e7373300fc8bb7c57cff7e4eda4fChris Lattner Value *Ptr = EmitScalarExpr(E->getArg(0)); 1346eb43f4a8f133c2bc510ae136a556e92b68a6ff44Eli Friedman QualType ElTy = E->getArg(0)->getType()->getPointeeType(); 1347eb43f4a8f133c2bc510ae136a556e92b68a6ff44Eli Friedman CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy); 1348ff993202abf6f5dc41c584c7103f5d39f248b3ddEli Friedman llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(), 1349ff993202abf6f5dc41c584c7103f5d39f248b3ddEli Friedman StoreSize.getQuantity() * 8); 1350ff993202abf6f5dc41c584c7103f5d39f248b3ddEli Friedman Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo()); 1351258f930227c1a102c9c22eee88df65f748863425Jim Grosbach llvm::StoreInst *Store = 135287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateAlignedStore(llvm::Constant::getNullValue(ITy), Ptr, 135387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar StoreSize); 13544967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Store->setAtomic(llvm::AtomicOrdering::Release); 13556bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 1356f58cd9bca9c2e7373300fc8bb7c57cff7e4eda4fChris Lattner } 1357ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar 1358f58cd9bca9c2e7373300fc8bb7c57cff7e4eda4fChris Lattner case Builtin::BI__sync_synchronize: { 1359c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman // We assume this is supposed to correspond to a C++0x-style 1360c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman // sequentially-consistent fence (i.e. this is only usable for 1361c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman // synchonization, not device I/O or anything like that). This intrinsic 1362258f930227c1a102c9c22eee88df65f748863425Jim Grosbach // is really badly designed in the sense that in theory, there isn't 1363c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman // any way to safely use it... but in practice, it mostly works 1364c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman // to use it with non-atomic loads and stores to get acquire/release 1365c83b975f1fb9d11e10b5aa25029ae9bb5fa80e07Eli Friedman // semantics. 13664967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent); 13676bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 1368f58cd9bca9c2e7373300fc8bb7c57cff7e4eda4fChris Lattner } 13691eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 137087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case Builtin::BI__builtin_nontemporal_load: 137187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return RValue::get(EmitNontemporalLoad(*this, E)); 137287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case Builtin::BI__builtin_nontemporal_store: 137387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return RValue::get(EmitNontemporalStore(*this, E)); 13742c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case Builtin::BI__c11_atomic_is_lock_free: 13752c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case Builtin::BI__atomic_is_lock_free: { 13762c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith // Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the 13772c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith // __c11 builtin, ptr is 0 (indicating a properly-aligned object), since 13782c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith // _Atomic(T) is always properly-aligned. 13792c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith const char *LibCallName = "__atomic_is_lock_free"; 13802c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith CallArgList Args; 13812c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Args.add(RValue::get(EmitScalarExpr(E->getArg(0))), 13822c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith getContext().getSizeType()); 13832c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith if (BuiltinID == Builtin::BI__atomic_is_lock_free) 13842c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Args.add(RValue::get(EmitScalarExpr(E->getArg(1))), 13852c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith getContext().VoidPtrTy); 13862c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith else 13872c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Args.add(RValue::get(llvm::Constant::getNullValue(VoidPtrTy)), 13882c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith getContext().VoidPtrTy); 13892c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith const CGFunctionInfo &FuncInfo = 13904967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar CGM.getTypes().arrangeBuiltinFunctionCall(E->getType(), Args); 13912c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo); 13922c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName); 13932c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith return EmitCall(FuncInfo, Func, ReturnValueSlot(), Args); 13942c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 13952c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 13962c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case Builtin::BI__atomic_test_and_set: { 13972c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith // Look at the argument type to determine whether this is a volatile 13982c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith // operation. The parameter type is always volatile. 13992c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType(); 14002c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith bool Volatile = 14012c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified(); 14022c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 14032c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Value *Ptr = EmitScalarExpr(E->getArg(0)); 1404956a5a17713deb1b5b27893303c4f308a1bd2a62Micah Villmow unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace(); 14052c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace)); 14062c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Value *NewVal = Builder.getInt8(1); 14072c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Value *Order = EmitScalarExpr(E->getArg(1)); 14082c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith if (isa<llvm::ConstantInt>(Order)) { 14092c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 14106bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines AtomicRMWInst *Result = nullptr; 14112c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith switch (ord) { 14122c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case 0: // memory_order_relaxed 14132c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith default: // invalid order 14144967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal, 14154967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering::Monotonic); 14162c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith break; 14174967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case 1: // memory_order_consume 14184967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case 2: // memory_order_acquire 14194967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal, 14204967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering::Acquire); 14212c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith break; 14224967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case 3: // memory_order_release 14234967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal, 14244967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering::Release); 14252c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith break; 14264967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case 4: // memory_order_acq_rel 14274967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 14284967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal, 14294967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering::AcquireRelease); 14302c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith break; 14314967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case 5: // memory_order_seq_cst 14324967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Result = Builder.CreateAtomicRMW( 14334967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicRMWInst::Xchg, Ptr, NewVal, 14344967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering::SequentiallyConsistent); 14352c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith break; 14362c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 14372c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Result->setVolatile(Volatile); 14382c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith return RValue::get(Builder.CreateIsNotNull(Result, "tobool")); 14392c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 14402c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 14412c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 14422c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 14432c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::BasicBlock *BBs[5] = { 14442c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith createBasicBlock("monotonic", CurFn), 14452c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith createBasicBlock("acquire", CurFn), 14462c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith createBasicBlock("release", CurFn), 14472c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith createBasicBlock("acqrel", CurFn), 14482c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith createBasicBlock("seqcst", CurFn) 14492c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith }; 14502c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::AtomicOrdering Orders[5] = { 14514967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Acquire, 14524967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering::Release, llvm::AtomicOrdering::AcquireRelease, 14534967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering::SequentiallyConsistent}; 14542c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 14552c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 14562c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]); 14572c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 14582c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Builder.SetInsertPoint(ContBB); 14592c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith PHINode *Result = Builder.CreatePHI(Int8Ty, 5, "was_set"); 14602c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 14612c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith for (unsigned i = 0; i < 5; ++i) { 14622c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Builder.SetInsertPoint(BBs[i]); 14632c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith AtomicRMWInst *RMW = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, 14642c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Ptr, NewVal, Orders[i]); 14652c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith RMW->setVolatile(Volatile); 14662c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Result->addIncoming(RMW, BBs[i]); 14672c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Builder.CreateBr(ContBB); 14682c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 14692c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 14702c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(0), BBs[0]); 14712c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(1), BBs[1]); 14722c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(2), BBs[1]); 14732c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(3), BBs[2]); 14742c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(4), BBs[3]); 14752c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(5), BBs[4]); 14762c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 14772c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Builder.SetInsertPoint(ContBB); 14782c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith return RValue::get(Builder.CreateIsNotNull(Result, "tobool")); 14792c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 14802c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 14812c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case Builtin::BI__atomic_clear: { 14822c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType(); 14832c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith bool Volatile = 14842c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified(); 14852c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 148687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Ptr = EmitPointerWithAlignment(E->getArg(0)); 148787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar unsigned AddrSpace = Ptr.getPointer()->getType()->getPointerAddressSpace(); 14882c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace)); 14892c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Value *NewVal = Builder.getInt8(0); 14902c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Value *Order = EmitScalarExpr(E->getArg(1)); 14912c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith if (isa<llvm::ConstantInt>(Order)) { 14922c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 14932c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile); 14942c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith switch (ord) { 14952c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case 0: // memory_order_relaxed 14962c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith default: // invalid order 14974967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Store->setOrdering(llvm::AtomicOrdering::Monotonic); 14982c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith break; 14992c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case 3: // memory_order_release 15004967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Store->setOrdering(llvm::AtomicOrdering::Release); 15012c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith break; 15022c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith case 5: // memory_order_seq_cst 15034967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Store->setOrdering(llvm::AtomicOrdering::SequentiallyConsistent); 15042c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith break; 15052c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 15066bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 15072c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 15082c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 15092c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 15102c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 15112c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::BasicBlock *BBs[3] = { 15122c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith createBasicBlock("monotonic", CurFn), 15132c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith createBasicBlock("release", CurFn), 15142c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith createBasicBlock("seqcst", CurFn) 15152c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith }; 15162c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::AtomicOrdering Orders[3] = { 15174967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Release, 15184967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering::SequentiallyConsistent}; 15192c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 15202c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 15212c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]); 15222c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 15232c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith for (unsigned i = 0; i < 3; ++i) { 15242c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Builder.SetInsertPoint(BBs[i]); 15252c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile); 15262c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Store->setOrdering(Orders[i]); 15272c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Builder.CreateBr(ContBB); 15282c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 15292c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 15302c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(0), BBs[0]); 15312c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(3), BBs[1]); 15322c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith SI->addCase(Builder.getInt32(5), BBs[2]); 15332c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 15342c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith Builder.SetInsertPoint(ContBB); 15356bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 15362c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith } 15372c39d71bb7cefdfe6116fa52454f3b3dc5abd517Richard Smith 1538276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman case Builtin::BI__atomic_thread_fence: 1539fafbf06732746f3ceca21d452d77b144ba8652aeRichard Smith case Builtin::BI__atomic_signal_fence: 1540fafbf06732746f3ceca21d452d77b144ba8652aeRichard Smith case Builtin::BI__c11_atomic_thread_fence: 1541fafbf06732746f3ceca21d452d77b144ba8652aeRichard Smith case Builtin::BI__c11_atomic_signal_fence: { 1542276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman llvm::SynchronizationScope Scope; 1543fafbf06732746f3ceca21d452d77b144ba8652aeRichard Smith if (BuiltinID == Builtin::BI__atomic_signal_fence || 1544fafbf06732746f3ceca21d452d77b144ba8652aeRichard Smith BuiltinID == Builtin::BI__c11_atomic_signal_fence) 1545276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Scope = llvm::SingleThread; 1546276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman else 1547276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Scope = llvm::CrossThread; 1548276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Value *Order = EmitScalarExpr(E->getArg(0)); 1549276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman if (isa<llvm::ConstantInt>(Order)) { 1550276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1551276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman switch (ord) { 1552276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman case 0: // memory_order_relaxed 1553276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman default: // invalid order 1554276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman break; 1555276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman case 1: // memory_order_consume 1556276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman case 2: // memory_order_acquire 15574967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Builder.CreateFence(llvm::AtomicOrdering::Acquire, Scope); 1558276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman break; 1559276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman case 3: // memory_order_release 15604967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Builder.CreateFence(llvm::AtomicOrdering::Release, Scope); 1561276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman break; 1562276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman case 4: // memory_order_acq_rel 15634967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, Scope); 1564276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman break; 1565276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman case 5: // memory_order_seq_cst 15664967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, 15674967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Scope); 1568276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman break; 1569276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman } 15706bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 1571276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman } 1572276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman 1573276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB; 1574276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman AcquireBB = createBasicBlock("acquire", CurFn); 1575276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman ReleaseBB = createBasicBlock("release", CurFn); 1576276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman AcqRelBB = createBasicBlock("acqrel", CurFn); 1577276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman SeqCstBB = createBasicBlock("seqcst", CurFn); 1578276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1579276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman 1580276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1581276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman llvm::SwitchInst *SI = Builder.CreateSwitch(Order, ContBB); 1582276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman 1583276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.SetInsertPoint(AcquireBB); 15844967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Builder.CreateFence(llvm::AtomicOrdering::Acquire, Scope); 1585276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateBr(ContBB); 1586276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman SI->addCase(Builder.getInt32(1), AcquireBB); 1587276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman SI->addCase(Builder.getInt32(2), AcquireBB); 1588276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman 1589276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.SetInsertPoint(ReleaseBB); 15904967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Builder.CreateFence(llvm::AtomicOrdering::Release, Scope); 1591276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateBr(ContBB); 1592276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman SI->addCase(Builder.getInt32(3), ReleaseBB); 1593276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman 1594276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.SetInsertPoint(AcqRelBB); 15954967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, Scope); 1596276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateBr(ContBB); 1597276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman SI->addCase(Builder.getInt32(4), AcqRelBB); 1598276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman 1599276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.SetInsertPoint(SeqCstBB); 16004967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, Scope); 1601276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.CreateBr(ContBB); 1602276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman SI->addCase(Builder.getInt32(5), SeqCstBB); 1603276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman 1604276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman Builder.SetInsertPoint(ContBB); 16056bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 1606276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman } 1607276b061970939293f1abaf694bd3ef05b2cbda79Eli Friedman 1608ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar // Library functions with special handling. 1609ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar case Builtin::BIsqrt: 1610ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar case Builtin::BIsqrtf: 1611ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar case Builtin::BIsqrtl: { 1612ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel // Transform a call to sqrt* into a @llvm.sqrt.* intrinsic call, but only 1613ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel // in finite- or unsafe-math mode (the intrinsic has different semantics 1614ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel // for handling negative numbers compared to the library function, so 1615ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel // -fmath-errno=0 is not enough). 1616ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel if (!FD->hasAttr<ConstAttr>()) 1617ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel break; 1618ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel if (!(CGM.getCodeGenOpts().UnsafeFPMath || 1619ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel CGM.getCodeGenOpts().NoNaNsFPMath)) 1620ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel break; 1621ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel Value *Arg0 = EmitScalarExpr(E->getArg(0)); 1622ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel llvm::Type *ArgType = Arg0->getType(); 1623ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel Value *F = CGM.getIntrinsic(Intrinsic::sqrt, ArgType); 1624ce4ad40100573008d013d2b9d3104c218ea34c2aHal Finkel return RValue::get(Builder.CreateCall(F, Arg0)); 1625ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar } 1626ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar 16270e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case Builtin::BI__builtin_pow: 16280e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case Builtin::BI__builtin_powf: 16290e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case Builtin::BI__builtin_powl: 1630ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar case Builtin::BIpow: 1631ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar case Builtin::BIpowf: 1632ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar case Builtin::BIpowl: { 16330323a7887b26f727080d2aa38f1a325dd23a1ee9Eli Bendersky // Transform a call to pow* into a @llvm.pow.* intrinsic call. 16340323a7887b26f727080d2aa38f1a325dd23a1ee9Eli Bendersky if (!FD->hasAttr<ConstAttr>()) 16350323a7887b26f727080d2aa38f1a325dd23a1ee9Eli Bendersky break; 16360323a7887b26f727080d2aa38f1a325dd23a1ee9Eli Bendersky Value *Base = EmitScalarExpr(E->getArg(0)); 16370323a7887b26f727080d2aa38f1a325dd23a1ee9Eli Bendersky Value *Exponent = EmitScalarExpr(E->getArg(1)); 16380323a7887b26f727080d2aa38f1a325dd23a1ee9Eli Bendersky llvm::Type *ArgType = Base->getType(); 16390323a7887b26f727080d2aa38f1a325dd23a1ee9Eli Bendersky Value *F = CGM.getIntrinsic(Intrinsic::pow, ArgType); 1640b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return RValue::get(Builder.CreateCall(F, {Base, Exponent})); 1641ef2abfee3ea16ec74942dc09e9e425f46aeb2582Daniel Dunbar } 1642ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman 1643094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich case Builtin::BIfma: 1644094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich case Builtin::BIfmaf: 1645094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich case Builtin::BIfmal: 1646094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich case Builtin::BI__builtin_fma: 1647094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich case Builtin::BI__builtin_fmaf: 1648094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich case Builtin::BI__builtin_fmal: { 1649094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich // Rewrite fma to intrinsic. 1650094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich Value *FirstArg = EmitScalarExpr(E->getArg(0)); 16519cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *ArgType = FirstArg->getType(); 16528dd55a3c3b28d195717c87bbc60e765951d408feBenjamin Kramer Value *F = CGM.getIntrinsic(Intrinsic::fma, ArgType); 1653b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return RValue::get( 1654b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Builder.CreateCall(F, {FirstArg, EmitScalarExpr(E->getArg(1)), 1655b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar EmitScalarExpr(E->getArg(2))})); 1656094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich } 1657094240ab184c3ca4b94e9d7eac80fcd34d8dd30cCameron Zwarich 1658ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman case Builtin::BI__builtin_signbit: 1659ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman case Builtin::BI__builtin_signbitf: 1660ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman case Builtin::BI__builtin_signbitl: { 16613ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar return RValue::get( 16623ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Builder.CreateZExt(EmitSignBit(*this, EmitScalarExpr(E->getArg(0))), 16633ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar ConvertType(E->getType()))); 1664ba68b08a05587490ed4c2e3d26f3a742c995c660Eli Friedman } 166577f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge case Builtin::BI__builtin_annotation: { 166677f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge llvm::Value *AnnVal = EmitScalarExpr(E->getArg(0)); 166777f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::annotation, 166877f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge AnnVal->getType()); 166977f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge 167077f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge // Get the annotation string, go through casts. Sema requires this to be a 167177f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge // non-wide string literal, potentially casted, so the cast<> is safe. 167277f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge const Expr *AnnotationStrExpr = E->getArg(1)->IgnoreParenCasts(); 1673cfa88f893915ceb8ae4ce2f17c46c24a4d67502fDmitri Gribenko StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString(); 167477f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge return RValue::get(EmitAnnotationCall(F, AnnVal, Str, E->getExprLoc())); 167577f68bb90af93b95045fb994e7cd68137adcc132Julien Lerouge } 1676ee76e7215d27c25bb053b71ae0a59119c08ccef9Michael Gottesman case Builtin::BI__builtin_addcb: 16770cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman case Builtin::BI__builtin_addcs: 16780cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman case Builtin::BI__builtin_addc: 16790cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman case Builtin::BI__builtin_addcl: 16807c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_addcll: 1681ee76e7215d27c25bb053b71ae0a59119c08ccef9Michael Gottesman case Builtin::BI__builtin_subcb: 16827c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_subcs: 16837c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_subc: 16847c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_subcl: 16857c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_subcll: { 16860cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman 16870cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // We translate all of these builtins from expressions of the form: 16880cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // int x = ..., y = ..., carryin = ..., carryout, result; 16890cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // result = __builtin_addc(x, y, carryin, &carryout); 16900cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // 16910cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // to LLVM IR of the form: 16920cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // 16930cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // %tmp1 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %x, i32 %y) 16940cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // %tmpsum1 = extractvalue {i32, i1} %tmp1, 0 16950cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // %carry1 = extractvalue {i32, i1} %tmp1, 1 16960cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // %tmp2 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %tmpsum1, 16970cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // i32 %carryin) 16980cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // %result = extractvalue {i32, i1} %tmp2, 0 16990cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // %carry2 = extractvalue {i32, i1} %tmp2, 1 17000cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // %tmp3 = or i1 %carry1, %carry2 17010cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // %tmp4 = zext i1 %tmp3 to i32 17020cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // store i32 %tmp4, i32* %carryout 17030cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman 17040cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // Scalarize our inputs. 17050cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *X = EmitScalarExpr(E->getArg(0)); 17060cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *Y = EmitScalarExpr(E->getArg(1)); 17070cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *Carryin = EmitScalarExpr(E->getArg(2)); 170887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address CarryOutPtr = EmitPointerWithAlignment(E->getArg(3)); 17090cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman 17107c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman // Decide if we are lowering to a uadd.with.overflow or usub.with.overflow. 17117c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman llvm::Intrinsic::ID IntrinsicId; 17127c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman switch (BuiltinID) { 17137c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman default: llvm_unreachable("Unknown multiprecision builtin id."); 1714ee76e7215d27c25bb053b71ae0a59119c08ccef9Michael Gottesman case Builtin::BI__builtin_addcb: 17157c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_addcs: 17167c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_addc: 17177c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_addcl: 17187c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_addcll: 17197c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman IntrinsicId = llvm::Intrinsic::uadd_with_overflow; 17207c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman break; 1721ee76e7215d27c25bb053b71ae0a59119c08ccef9Michael Gottesman case Builtin::BI__builtin_subcb: 17227c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_subcs: 17237c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_subc: 17247c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_subcl: 17257c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman case Builtin::BI__builtin_subcll: 17267c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman IntrinsicId = llvm::Intrinsic::usub_with_overflow; 17277c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman break; 17287c9dbb76f6847fb30527c8e74ef9a25a3d4eb731Michael Gottesman } 17290cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman 17300cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman // Construct our resulting LLVM IR expression. 17310cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *Carry1; 17320cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *Sum1 = EmitOverflowIntrinsic(*this, IntrinsicId, 17330cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman X, Y, Carry1); 17340cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *Carry2; 17350cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *Sum2 = EmitOverflowIntrinsic(*this, IntrinsicId, 17360cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman Sum1, Carryin, Carry2); 17370cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman llvm::Value *CarryOut = Builder.CreateZExt(Builder.CreateOr(Carry1, Carry2), 17380cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman X->getType()); 173987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateStore(CarryOut, CarryOutPtr); 17400cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman return RValue::get(Sum2); 17410cf07bcc6c53f1df421d4c40ce52d43e8d8adcd3Michael Gottesman } 174287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 174387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case Builtin::BI__builtin_add_overflow: 174487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case Builtin::BI__builtin_sub_overflow: 174587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case Builtin::BI__builtin_mul_overflow: { 174687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar const clang::Expr *LeftArg = E->getArg(0); 174787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar const clang::Expr *RightArg = E->getArg(1); 174887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar const clang::Expr *ResultArg = E->getArg(2); 174987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 175087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar clang::QualType ResultQTy = 175187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar ResultArg->getType()->castAs<PointerType>()->getPointeeType(); 175287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 175387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar WidthAndSignedness LeftInfo = 175487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar getIntegerWidthAndSignedness(CGM.getContext(), LeftArg->getType()); 175587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar WidthAndSignedness RightInfo = 175687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar getIntegerWidthAndSignedness(CGM.getContext(), RightArg->getType()); 175787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar WidthAndSignedness ResultInfo = 175887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar getIntegerWidthAndSignedness(CGM.getContext(), ResultQTy); 175987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar WidthAndSignedness EncompassingInfo = 176087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar EncompassingIntegerType({LeftInfo, RightInfo, ResultInfo}); 176187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 176287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *EncompassingLLVMTy = 176387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::IntegerType::get(CGM.getLLVMContext(), EncompassingInfo.Width); 176487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 176587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *ResultLLVMTy = CGM.getTypes().ConvertType(ResultQTy); 176687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 176787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Intrinsic::ID IntrinsicId; 176887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar switch (BuiltinID) { 176987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar default: 177087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm_unreachable("Unknown overflow builtin id."); 177187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case Builtin::BI__builtin_add_overflow: 177287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar IntrinsicId = EncompassingInfo.Signed 177387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar ? llvm::Intrinsic::sadd_with_overflow 177487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar : llvm::Intrinsic::uadd_with_overflow; 177587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar break; 177687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case Builtin::BI__builtin_sub_overflow: 177787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar IntrinsicId = EncompassingInfo.Signed 177887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar ? llvm::Intrinsic::ssub_with_overflow 177987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar : llvm::Intrinsic::usub_with_overflow; 178087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar break; 178187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case Builtin::BI__builtin_mul_overflow: 178287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar IntrinsicId = EncompassingInfo.Signed 178387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar ? llvm::Intrinsic::smul_with_overflow 178487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar : llvm::Intrinsic::umul_with_overflow; 178587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar break; 178687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 178787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 178887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Value *Left = EmitScalarExpr(LeftArg); 178987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Value *Right = EmitScalarExpr(RightArg); 179087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address ResultPtr = EmitPointerWithAlignment(ResultArg); 179187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 179287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Extend each operand to the encompassing type. 179387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Left = Builder.CreateIntCast(Left, EncompassingLLVMTy, LeftInfo.Signed); 179487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Right = Builder.CreateIntCast(Right, EncompassingLLVMTy, RightInfo.Signed); 179587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 179687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Perform the operation on the extended values. 179787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Value *Overflow, *Result; 179887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Result = EmitOverflowIntrinsic(*this, IntrinsicId, Left, Right, Overflow); 179987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 180087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar if (EncompassingInfo.Width > ResultInfo.Width) { 180187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // The encompassing type is wider than the result type, so we need to 180287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // truncate it. 180387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Value *ResultTrunc = Builder.CreateTrunc(Result, ResultLLVMTy); 180487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 180587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // To see if the truncation caused an overflow, we will extend 180687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // the result and then compare it to the original result. 180787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Value *ResultTruncExt = Builder.CreateIntCast( 180887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar ResultTrunc, EncompassingLLVMTy, ResultInfo.Signed); 180987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Value *TruncationOverflow = 181087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateICmpNE(Result, ResultTruncExt); 181187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 181287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Overflow = Builder.CreateOr(Overflow, TruncationOverflow); 181387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Result = ResultTrunc; 181487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 181587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 181687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Finally, store the result using the pointer. 181787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar bool isVolatile = 181887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar ResultArg->getType()->getPointeeType().isVolatileQualified(); 181987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateStore(EmitToMemory(Result, ResultQTy), ResultPtr, isVolatile); 182087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 182187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return RValue::get(Overflow); 182287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 182387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 182498d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_uadd_overflow: 182598d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_uaddl_overflow: 182698d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_uaddll_overflow: 182798d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_usub_overflow: 182898d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_usubl_overflow: 182998d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_usubll_overflow: 183098d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_umul_overflow: 183198d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_umull_overflow: 183298d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_umulll_overflow: 183398d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_sadd_overflow: 183498d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_saddl_overflow: 183598d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_saddll_overflow: 183698d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_ssub_overflow: 183798d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_ssubl_overflow: 183898d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_ssubll_overflow: 183998d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_smul_overflow: 184098d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_smull_overflow: 184198d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_smulll_overflow: { 184298d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman 184398d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman // We translate all of these builtins directly to the relevant llvm IR node. 184498d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman 184598d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman // Scalarize our inputs. 184698d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman llvm::Value *X = EmitScalarExpr(E->getArg(0)); 184798d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman llvm::Value *Y = EmitScalarExpr(E->getArg(1)); 184887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address SumOutPtr = EmitPointerWithAlignment(E->getArg(2)); 184998d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman 185098d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman // Decide which of the overflow intrinsics we are lowering to: 185198d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman llvm::Intrinsic::ID IntrinsicId; 185298d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman switch (BuiltinID) { 185387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar default: llvm_unreachable("Unknown overflow builtin id."); 185498d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_uadd_overflow: 185598d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_uaddl_overflow: 185698d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_uaddll_overflow: 185798d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman IntrinsicId = llvm::Intrinsic::uadd_with_overflow; 185898d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman break; 185998d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_usub_overflow: 186098d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_usubl_overflow: 186198d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_usubll_overflow: 186298d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman IntrinsicId = llvm::Intrinsic::usub_with_overflow; 186398d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman break; 186498d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_umul_overflow: 186598d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_umull_overflow: 186698d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_umulll_overflow: 186798d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman IntrinsicId = llvm::Intrinsic::umul_with_overflow; 186898d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman break; 186998d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_sadd_overflow: 187098d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_saddl_overflow: 187198d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_saddll_overflow: 187298d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman IntrinsicId = llvm::Intrinsic::sadd_with_overflow; 187398d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman break; 187498d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_ssub_overflow: 187598d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_ssubl_overflow: 187698d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_ssubll_overflow: 187798d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman IntrinsicId = llvm::Intrinsic::ssub_with_overflow; 187898d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman break; 187998d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_smul_overflow: 188098d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_smull_overflow: 188198d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman case Builtin::BI__builtin_smulll_overflow: 188298d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman IntrinsicId = llvm::Intrinsic::smul_with_overflow; 188398d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman break; 188498d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman } 188598d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman 18864967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 188798d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman llvm::Value *Carry; 188898d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman llvm::Value *Sum = EmitOverflowIntrinsic(*this, IntrinsicId, X, Y, Carry); 188987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateStore(Sum, SumOutPtr); 189098d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman 189198d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman return RValue::get(Carry); 189298d1ec1e99625176626b0bcd44cef7df6e89b289Michael Gottesman } 18935154dce6388e3aaa445467030df7a45ed1211abeRichard Smith case Builtin::BI__builtin_addressof: 189487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return RValue::get(EmitLValue(E->getArg(0)).getPointer()); 1895c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case Builtin::BI__builtin_operator_new: 1896c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines return EmitBuiltinNewDeleteCall(FD->getType()->castAs<FunctionProtoType>(), 1897c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines E->getArg(0), false); 1898c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case Builtin::BI__builtin_operator_delete: 1899c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines return EmitBuiltinNewDeleteCall(FD->getType()->castAs<FunctionProtoType>(), 1900c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines E->getArg(0), true); 190127844533aad5c165523a5926424097699610d3f0Nico Weber case Builtin::BI__noop: 1902176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // __noop always evaluates to an integer literal zero. 1903176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return RValue::get(ConstantInt::get(IntTy, 0)); 19040e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case Builtin::BI__builtin_call_with_static_chain: { 19050e2c34f92f00628d48968dfea096d36381f494cbStephen Hines const CallExpr *Call = cast<CallExpr>(E->getArg(0)); 19060e2c34f92f00628d48968dfea096d36381f494cbStephen Hines const Expr *Chain = E->getArg(1); 19070e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return EmitCall(Call->getCallee()->getType(), 19080e2c34f92f00628d48968dfea096d36381f494cbStephen Hines EmitScalarExpr(Call->getCallee()), Call, ReturnValue, 19090e2c34f92f00628d48968dfea096d36381f494cbStephen Hines Call->getCalleeDecl(), EmitScalarExpr(Chain)); 19100e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 1911c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case Builtin::BI_InterlockedExchange: 1912c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case Builtin::BI_InterlockedExchangePointer: 1913c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E); 1914c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case Builtin::BI_InterlockedCompareExchangePointer: { 1915c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Type *RTy; 1916c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::IntegerType *IntType = 1917c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines IntegerType::get(getLLVMContext(), 1918c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines getContext().getTypeSize(E->getType())); 1919c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Type *IntPtrType = IntType->getPointerTo(); 1920c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 1921c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Destination = 1922c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), IntPtrType); 1923c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 1924c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Exchange = EmitScalarExpr(E->getArg(1)); 1925c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines RTy = Exchange->getType(); 1926c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Exchange = Builder.CreatePtrToInt(Exchange, IntType); 1927c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 1928c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Comparand = 1929c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Builder.CreatePtrToInt(EmitScalarExpr(E->getArg(2)), IntType); 1930c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 19314967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar auto Result = 19324967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange, 19334967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar AtomicOrdering::SequentiallyConsistent, 19344967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar AtomicOrdering::SequentiallyConsistent); 1935c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Result->setVolatile(true); 1936c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 1937c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines return RValue::get(Builder.CreateIntToPtr(Builder.CreateExtractValue(Result, 1938c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 0), 1939c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines RTy)); 1940c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines } 1941651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case Builtin::BI_InterlockedCompareExchange: { 1942651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AtomicCmpXchgInst *CXI = Builder.CreateAtomicCmpXchg( 1943651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines EmitScalarExpr(E->getArg(0)), 1944651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines EmitScalarExpr(E->getArg(2)), 1945651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines EmitScalarExpr(E->getArg(1)), 19464967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar AtomicOrdering::SequentiallyConsistent, 19474967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar AtomicOrdering::SequentiallyConsistent); 1948651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CXI->setVolatile(true); 1949c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines return RValue::get(Builder.CreateExtractValue(CXI, 0)); 1950651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 1951651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case Builtin::BI_InterlockedIncrement: { 19524967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Type *IntTy = ConvertType(E->getType()); 1953651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( 1954651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AtomicRMWInst::Add, 1955651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines EmitScalarExpr(E->getArg(0)), 19564967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar ConstantInt::get(IntTy, 1), 19574967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering::SequentiallyConsistent); 1958651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines RMWI->setVolatile(true); 19594967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get(Builder.CreateAdd(RMWI, ConstantInt::get(IntTy, 1))); 1960651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 1961651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case Builtin::BI_InterlockedDecrement: { 19624967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Type *IntTy = ConvertType(E->getType()); 1963651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( 1964651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AtomicRMWInst::Sub, 1965651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines EmitScalarExpr(E->getArg(0)), 19664967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar ConstantInt::get(IntTy, 1), 19674967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering::SequentiallyConsistent); 1968651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines RMWI->setVolatile(true); 19694967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get(Builder.CreateSub(RMWI, ConstantInt::get(IntTy, 1))); 1970651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 1971651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case Builtin::BI_InterlockedExchangeAdd: { 1972651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( 1973651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AtomicRMWInst::Add, 1974651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines EmitScalarExpr(E->getArg(0)), 1975651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines EmitScalarExpr(E->getArg(1)), 19764967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering::SequentiallyConsistent); 1977651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines RMWI->setVolatile(true); 1978651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return RValue::get(RMWI); 1979651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 1980176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case Builtin::BI__readfsdword: { 19814967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Type *IntTy = ConvertType(E->getType()); 1982176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *IntToPtr = 1983176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Builder.CreateIntToPtr(EmitScalarExpr(E->getArg(0)), 19844967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::PointerType::get(IntTy, 257)); 1985176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines LoadInst *Load = 19864967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Builder.CreateDefaultAlignedLoad(IntToPtr, /*isVolatile=*/true); 1987176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return RValue::get(Load); 1988176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 19890e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 19900e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case Builtin::BI__exception_code: 19910e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case Builtin::BI_exception_code: 19920e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return RValue::get(EmitSEHExceptionCode()); 19930e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case Builtin::BI__exception_info: 19940e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case Builtin::BI_exception_info: 19950e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return RValue::get(EmitSEHExceptionInfo()); 19960e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case Builtin::BI__abnormal_termination: 19970e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case Builtin::BI_abnormal_termination: 19980e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return RValue::get(EmitSEHAbnormalTermination()); 19990e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case Builtin::BI_setjmpex: { 20000e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (getTarget().getTriple().isOSMSVCRT()) { 20010e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Type *ArgTypes[] = {Int8PtrTy, Int8PtrTy}; 20020e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::AttributeSet ReturnsTwiceAttr = 20030e2c34f92f00628d48968dfea096d36381f494cbStephen Hines AttributeSet::get(getLLVMContext(), llvm::AttributeSet::FunctionIndex, 20040e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Attribute::ReturnsTwice); 20050e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Constant *SetJmpEx = CGM.CreateRuntimeFunction( 20060e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/false), 20070e2c34f92f00628d48968dfea096d36381f494cbStephen Hines "_setjmpex", ReturnsTwiceAttr); 20083ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar llvm::Value *Buf = Builder.CreateBitOrPointerCast( 20093ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar EmitScalarExpr(E->getArg(0)), Int8PtrTy); 20100e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Value *FrameAddr = 20110e2c34f92f00628d48968dfea096d36381f494cbStephen Hines Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress), 20120e2c34f92f00628d48968dfea096d36381f494cbStephen Hines ConstantInt::get(Int32Ty, 0)); 20130e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Value *Args[] = {Buf, FrameAddr}; 20140e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::CallSite CS = EmitRuntimeCallOrInvoke(SetJmpEx, Args); 20150e2c34f92f00628d48968dfea096d36381f494cbStephen Hines CS.setAttributes(ReturnsTwiceAttr); 20160e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return RValue::get(CS.getInstruction()); 20170e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 20183ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar break; 20190e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 20200e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case Builtin::BI_setjmp: { 20210e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (getTarget().getTriple().isOSMSVCRT()) { 20220e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::AttributeSet ReturnsTwiceAttr = 20230e2c34f92f00628d48968dfea096d36381f494cbStephen Hines AttributeSet::get(getLLVMContext(), llvm::AttributeSet::FunctionIndex, 20240e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Attribute::ReturnsTwice); 20253ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar llvm::Value *Buf = Builder.CreateBitOrPointerCast( 20263ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar EmitScalarExpr(E->getArg(0)), Int8PtrTy); 20270e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::CallSite CS; 20280e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (getTarget().getTriple().getArch() == llvm::Triple::x86) { 20290e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Type *ArgTypes[] = {Int8PtrTy, IntTy}; 20300e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Constant *SetJmp3 = CGM.CreateRuntimeFunction( 20310e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/true), 20320e2c34f92f00628d48968dfea096d36381f494cbStephen Hines "_setjmp3", ReturnsTwiceAttr); 20330e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Value *Count = ConstantInt::get(IntTy, 0); 20340e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Value *Args[] = {Buf, Count}; 20350e2c34f92f00628d48968dfea096d36381f494cbStephen Hines CS = EmitRuntimeCallOrInvoke(SetJmp3, Args); 20360e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } else { 20370e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Type *ArgTypes[] = {Int8PtrTy, Int8PtrTy}; 20380e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Constant *SetJmp = CGM.CreateRuntimeFunction( 20390e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/false), 20400e2c34f92f00628d48968dfea096d36381f494cbStephen Hines "_setjmp", ReturnsTwiceAttr); 20410e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Value *FrameAddr = 20420e2c34f92f00628d48968dfea096d36381f494cbStephen Hines Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress), 20430e2c34f92f00628d48968dfea096d36381f494cbStephen Hines ConstantInt::get(Int32Ty, 0)); 20440e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Value *Args[] = {Buf, FrameAddr}; 20450e2c34f92f00628d48968dfea096d36381f494cbStephen Hines CS = EmitRuntimeCallOrInvoke(SetJmp, Args); 20460e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 20470e2c34f92f00628d48968dfea096d36381f494cbStephen Hines CS.setAttributes(ReturnsTwiceAttr); 20480e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return RValue::get(CS.getInstruction()); 20490e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 20503ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar break; 20513ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar } 20523ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 20533ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar case Builtin::BI__GetExceptionInfo: { 20543ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (llvm::GlobalVariable *GV = 20553ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar CGM.getCXXABI().getThrowInfo(FD->getParamDecl(0)->getType())) 20563ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar return RValue::get(llvm::ConstantExpr::getBitCast(GV, CGM.Int8PtrTy)); 20573ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar break; 20580e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 20594967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 20604967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // OpenCL v2.0 s6.13.16.2, Built-in pipe read and write functions 20614967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BIread_pipe: 20624967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BIwrite_pipe: { 20634967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Arg0 = EmitScalarExpr(E->getArg(0)), 20644967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar *Arg1 = EmitScalarExpr(E->getArg(1)); 20654967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 20664967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Type of the generic packet parameter. 20674967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar unsigned GenericAS = 20684967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar getContext().getTargetAddressSpace(LangAS::opencl_generic); 20694967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Type *I8PTy = llvm::PointerType::get( 20704967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Type::getInt8Ty(getLLVMContext()), GenericAS); 20714967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 20724967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Testing which overloaded version we should generate the call for. 20734967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (2U == E->getNumArgs()) { 20744967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_2" 20754967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar : "__write_pipe_2"; 20764967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Creating a generic function type to be able to call with any builtin or 20774967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // user defined type. 20784967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Type *ArgTys[] = {Arg0->getType(), I8PTy}; 20794967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::FunctionType *FTy = llvm::FunctionType::get( 20804967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false); 20814967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *BCast = Builder.CreatePointerCast(Arg1, I8PTy); 20824967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get(Builder.CreateCall( 20834967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar CGM.CreateRuntimeFunction(FTy, Name), {Arg0, BCast})); 20844967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } else { 20854967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar assert(4 == E->getNumArgs() && 20864967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar "Illegal number of parameters to pipe function"); 20874967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_4" 20884967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar : "__write_pipe_4"; 20894967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 20904967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, I8PTy}; 20914967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Arg2 = EmitScalarExpr(E->getArg(2)), 20924967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar *Arg3 = EmitScalarExpr(E->getArg(3)); 20934967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::FunctionType *FTy = llvm::FunctionType::get( 20944967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false); 20954967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *BCast = Builder.CreatePointerCast(Arg3, I8PTy); 20964967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // We know the third argument is an integer type, but we may need to cast 20974967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // it to i32. 20984967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (Arg2->getType() != Int32Ty) 20994967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Arg2 = Builder.CreateZExtOrTrunc(Arg2, Int32Ty); 21004967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get(Builder.CreateCall( 21014967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar CGM.CreateRuntimeFunction(FTy, Name), {Arg0, Arg1, Arg2, BCast})); 21024967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 21034967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 21044967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // OpenCL v2.0 s6.13.16 ,s9.17.3.5 - Built-in pipe reserve read and write 21054967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // functions 21064967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BIreserve_read_pipe: 21074967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BIreserve_write_pipe: 21084967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BIwork_group_reserve_read_pipe: 21094967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BIwork_group_reserve_write_pipe: 21104967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BIsub_group_reserve_read_pipe: 21114967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BIsub_group_reserve_write_pipe: { 21124967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Composing the mangled name for the function. 21134967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar const char *Name; 21144967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (BuiltinID == Builtin::BIreserve_read_pipe) 21154967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Name = "__reserve_read_pipe"; 21164967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar else if (BuiltinID == Builtin::BIreserve_write_pipe) 21174967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Name = "__reserve_write_pipe"; 21184967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe) 21194967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Name = "__work_group_reserve_read_pipe"; 21204967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe) 21214967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Name = "__work_group_reserve_write_pipe"; 21224967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe) 21234967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Name = "__sub_group_reserve_read_pipe"; 21244967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar else 21254967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Name = "__sub_group_reserve_write_pipe"; 21264967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 21274967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Arg0 = EmitScalarExpr(E->getArg(0)), 21284967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar *Arg1 = EmitScalarExpr(E->getArg(1)); 21294967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Type *ReservedIDTy = ConvertType(getContext().OCLReserveIDTy); 21304967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 21314967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Building the generic function prototype. 21324967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty}; 21334967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::FunctionType *FTy = llvm::FunctionType::get( 21344967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar ReservedIDTy, llvm::ArrayRef<llvm::Type *>(ArgTys), false); 21354967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // We know the second argument is an integer type, but we may need to cast 21364967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // it to i32. 21374967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (Arg1->getType() != Int32Ty) 21384967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Arg1 = Builder.CreateZExtOrTrunc(Arg1, Int32Ty); 21394967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get( 21404967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), {Arg0, Arg1})); 21414967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 21424967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // OpenCL v2.0 s6.13.16, s9.17.3.5 - Built-in pipe commit read and write 21434967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // functions 21444967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BIcommit_read_pipe: 21454967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BIcommit_write_pipe: 21464967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BIwork_group_commit_read_pipe: 21474967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BIwork_group_commit_write_pipe: 21484967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BIsub_group_commit_read_pipe: 21494967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BIsub_group_commit_write_pipe: { 21504967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar const char *Name; 21514967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (BuiltinID == Builtin::BIcommit_read_pipe) 21524967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Name = "__commit_read_pipe"; 21534967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar else if (BuiltinID == Builtin::BIcommit_write_pipe) 21544967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Name = "__commit_write_pipe"; 21554967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe) 21564967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Name = "__work_group_commit_read_pipe"; 21574967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe) 21584967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Name = "__work_group_commit_write_pipe"; 21594967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe) 21604967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Name = "__sub_group_commit_read_pipe"; 21614967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar else 21624967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Name = "__sub_group_commit_write_pipe"; 21634967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 21644967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Arg0 = EmitScalarExpr(E->getArg(0)), 21654967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar *Arg1 = EmitScalarExpr(E->getArg(1)); 21664967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 21674967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Building the generic function prototype. 21684967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType()}; 21694967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::FunctionType *FTy = 21704967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::FunctionType::get(llvm::Type::getVoidTy(getLLVMContext()), 21714967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::ArrayRef<llvm::Type *>(ArgTys), false); 21724967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 21734967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get( 21744967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), {Arg0, Arg1})); 21754967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 21764967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // OpenCL v2.0 s6.13.16.4 Built-in pipe query functions 21774967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BIget_pipe_num_packets: 21784967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BIget_pipe_max_packets: { 21794967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar const char *Name; 21804967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (BuiltinID == Builtin::BIget_pipe_num_packets) 21814967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Name = "__get_pipe_num_packets"; 21824967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar else 21834967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Name = "__get_pipe_max_packets"; 21844967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 21854967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Building the generic function prototype. 21864967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Arg0 = EmitScalarExpr(E->getArg(0)); 21874967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Type *ArgTys[] = {Arg0->getType()}; 21884967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::FunctionType *FTy = llvm::FunctionType::get( 21894967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false); 21904967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 21914967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get( 21924967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), {Arg0})); 21934967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 21944967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 21954967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // OpenCL v2.0 s6.13.9 - Address space qualifier functions. 21964967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BIto_global: 21974967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BIto_local: 21984967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BIto_private: { 21994967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar auto Arg0 = EmitScalarExpr(E->getArg(0)); 22004967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar auto NewArgT = llvm::PointerType::get(Int8Ty, 22014967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar CGM.getContext().getTargetAddressSpace(LangAS::opencl_generic)); 22024967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar auto NewRetT = llvm::PointerType::get(Int8Ty, 22034967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar CGM.getContext().getTargetAddressSpace( 22044967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar E->getType()->getPointeeType().getAddressSpace())); 22054967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar auto FTy = llvm::FunctionType::get(NewRetT, {NewArgT}, false); 22064967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Value *NewArg; 22074967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (Arg0->getType()->getPointerAddressSpace() != 22084967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar NewArgT->getPointerAddressSpace()) 22094967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar NewArg = Builder.CreateAddrSpaceCast(Arg0, NewArgT); 22104967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar else 22114967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar NewArg = Builder.CreateBitOrPointerCast(Arg0, NewArgT); 22124967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar auto NewCall = Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, 22134967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar E->getDirectCallee()->getName()), {NewArg}); 22144967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get(Builder.CreateBitOrPointerCast(NewCall, 22154967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar ConvertType(E->getType()))); 22164967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 22174967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 22184967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // OpenCL v2.0, s6.13.17 - Enqueue kernel function. 22194967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // It contains four different overload formats specified in Table 6.13.17.1. 22204967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BIenqueue_kernel: { 22214967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar StringRef Name; // Generated function call name 22224967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar unsigned NumArgs = E->getNumArgs(); 22234967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 22244967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Type *QueueTy = ConvertType(getContext().OCLQueueTy); 22254967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Type *RangeTy = ConvertType(getContext().OCLNDRangeTy); 22264967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 22274967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Value *Queue = EmitScalarExpr(E->getArg(0)); 22284967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Value *Flags = EmitScalarExpr(E->getArg(1)); 22294967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Value *Range = EmitScalarExpr(E->getArg(2)); 22304967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 22314967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (NumArgs == 4) { 22324967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // The most basic form of the call with parameters: 22334967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // queue_t, kernel_enqueue_flags_t, ndrange_t, block(void) 22344967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Name = "__enqueue_kernel_basic"; 22354967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Type *ArgTys[] = {QueueTy, Int32Ty, RangeTy, Int8PtrTy}; 22364967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::FunctionType *FTy = llvm::FunctionType::get( 22374967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys, 4), false); 22384967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 22394967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Value *Block = 22404967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Builder.CreateBitCast(EmitScalarExpr(E->getArg(3)), Int8PtrTy); 22414967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 22424967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get(Builder.CreateCall( 22434967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar CGM.CreateRuntimeFunction(FTy, Name), {Queue, Flags, Range, Block})); 22444967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 22454967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar assert(NumArgs >= 5 && "Invalid enqueue_kernel signature"); 22464967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 22474967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Could have events and/or vaargs. 22484967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (E->getArg(3)->getType()->isBlockPointerType()) { 22494967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // No events passed, but has variadic arguments. 22504967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Name = "__enqueue_kernel_vaargs"; 22514967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Value *Block = 22524967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Builder.CreateBitCast(EmitScalarExpr(E->getArg(3)), Int8PtrTy); 22534967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Create a vector of the arguments, as well as a constant value to 22544967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // express to the runtime the number of variadic arguments. 22554967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar std::vector<llvm::Value *> Args = {Queue, Flags, Range, Block, 22564967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar ConstantInt::get(IntTy, NumArgs - 4)}; 22574967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar std::vector<llvm::Type *> ArgTys = {QueueTy, IntTy, RangeTy, Int8PtrTy, 22584967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar IntTy}; 22594967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 22604967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Add the variadics. 22614967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar for (unsigned I = 4; I < NumArgs; ++I) { 22624967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Value *ArgSize = EmitScalarExpr(E->getArg(I)); 22634967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar unsigned TypeSizeInBytes = 22644967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar getContext() 22654967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar .getTypeSizeInChars(E->getArg(I)->getType()) 22664967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar .getQuantity(); 22674967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Args.push_back(TypeSizeInBytes < 4 22684967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar ? Builder.CreateZExt(ArgSize, Int32Ty) 22694967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar : ArgSize); 22704967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 22714967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 22724967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::FunctionType *FTy = llvm::FunctionType::get( 22734967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), true); 22744967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get( 22754967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), 22764967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::ArrayRef<llvm::Value *>(Args))); 22774967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 22784967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Any calls now have event arguments passed. 22794967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (NumArgs >= 7) { 22804967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Type *EventTy = ConvertType(getContext().OCLClkEventTy); 22814967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar unsigned AS4 = 22824967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar E->getArg(4)->getType()->isArrayType() 22834967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar ? E->getArg(4)->getType().getAddressSpace() 22844967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar : E->getArg(4)->getType()->getPointeeType().getAddressSpace(); 22854967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Type *EventPtrAS4Ty = 22864967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar EventTy->getPointerTo(CGM.getContext().getTargetAddressSpace(AS4)); 22874967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar unsigned AS5 = 22884967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar E->getArg(5)->getType()->getPointeeType().getAddressSpace(); 22894967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Type *EventPtrAS5Ty = 22904967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar EventTy->getPointerTo(CGM.getContext().getTargetAddressSpace(AS5)); 22914967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 22924967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Value *NumEvents = EmitScalarExpr(E->getArg(3)); 22934967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Value *EventList = 22944967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar E->getArg(4)->getType()->isArrayType() 22954967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar ? EmitArrayToPointerDecay(E->getArg(4)).getPointer() 22964967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar : EmitScalarExpr(E->getArg(4)); 22974967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Value *ClkEvent = EmitScalarExpr(E->getArg(5)); 22984967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Value *Block = 22994967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Builder.CreateBitCast(EmitScalarExpr(E->getArg(6)), Int8PtrTy); 23004967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 23014967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar std::vector<llvm::Type *> ArgTys = { 23024967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar QueueTy, Int32Ty, RangeTy, Int32Ty, 23034967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar EventPtrAS4Ty, EventPtrAS5Ty, Int8PtrTy}; 23044967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar std::vector<llvm::Value *> Args = {Queue, Flags, Range, NumEvents, 23054967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar EventList, ClkEvent, Block}; 23064967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 23074967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (NumArgs == 7) { 23084967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Has events but no variadics. 23094967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Name = "__enqueue_kernel_basic_events"; 23104967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::FunctionType *FTy = llvm::FunctionType::get( 23114967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false); 23124967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get( 23134967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), 23144967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::ArrayRef<llvm::Value *>(Args))); 23154967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 23164967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Has event info and variadics 23174967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Pass the number of variadics to the runtime function too. 23184967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Args.push_back(ConstantInt::get(Int32Ty, NumArgs - 7)); 23194967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar ArgTys.push_back(Int32Ty); 23204967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Name = "__enqueue_kernel_events_vaargs"; 23214967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 23224967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Add the variadics. 23234967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar for (unsigned I = 7; I < NumArgs; ++I) { 23244967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Value *ArgSize = EmitScalarExpr(E->getArg(I)); 23254967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar unsigned TypeSizeInBytes = 23264967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar getContext() 23274967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar .getTypeSizeInChars(E->getArg(I)->getType()) 23284967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar .getQuantity(); 23294967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Args.push_back(TypeSizeInBytes < 4 23304967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar ? Builder.CreateZExt(ArgSize, Int32Ty) 23314967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar : ArgSize); 23324967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 23334967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::FunctionType *FTy = llvm::FunctionType::get( 23344967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), true); 23354967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get( 23364967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), 23374967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::ArrayRef<llvm::Value *>(Args))); 23384967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 23394967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 23404967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // OpenCL v2.0 s6.13.17.6 - Kernel query functions need bitcast of block 23414967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // parameter. 23424967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BIget_kernel_work_group_size: { 23434967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Arg = EmitScalarExpr(E->getArg(0)); 23444967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Arg = Builder.CreateBitCast(Arg, Int8PtrTy); 23454967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get( 23464967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Builder.CreateCall(CGM.CreateRuntimeFunction( 23474967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::FunctionType::get(IntTy, Int8PtrTy, false), 23484967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar "__get_kernel_work_group_size_impl"), 23494967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Arg)); 23504967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 23514967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BIget_kernel_preferred_work_group_size_multiple: { 23524967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Arg = EmitScalarExpr(E->getArg(0)); 23534967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Arg = Builder.CreateBitCast(Arg, Int8PtrTy); 23544967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get(Builder.CreateCall( 23554967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar CGM.CreateRuntimeFunction( 23564967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::FunctionType::get(IntTy, Int8PtrTy, false), 23574967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar "__get_kernel_preferred_work_group_multiple_impl"), 23584967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Arg)); 23594967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 23604967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BIprintf: 23614967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (getLangOpts().CUDA && getLangOpts().CUDAIsDevice) 23624967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return EmitCUDADevicePrintfCallExpr(E, ReturnValue); 23634967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar break; 23644967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_canonicalize: 23654967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_canonicalizef: 23664967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_canonicalizel: 23674967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::canonicalize)); 23684967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 23694967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case Builtin::BI__builtin_thread_pointer: { 23704967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (!getContext().getTargetInfo().isTLSSupported()) 23714967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar CGM.ErrorUnsupported(E, "__builtin_thread_pointer"); 23724967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Fall through - it's already mapped to the intrinsic by GCCBuiltin. 23734967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar break; 23744967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 23757ea2e3f6aae9b7511686d3d26dee690fee81c3aaNate Begeman } 23761eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2377a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // If this is an alias for a lib function (e.g. __builtin_sin), emit 2378a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // the call using the normal call path, but using the unmangled 2379a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // version of the function name. 2380a45680b7e7c49ea9893c6cff585984f3e4120366John McCall if (getContext().BuiltinInfo.isLibFunction(BuiltinID)) 2381a45680b7e7c49ea9893c6cff585984f3e4120366John McCall return emitLibraryCall(*this, FD, E, 2382a45680b7e7c49ea9893c6cff585984f3e4120366John McCall CGM.getBuiltinLibFunction(FD, BuiltinID)); 2383258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 2384a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // If this is a predefined lib function (e.g. malloc), emit the call 2385a45680b7e7c49ea9893c6cff585984f3e4120366John McCall // using exactly the normal call path. 2386a45680b7e7c49ea9893c6cff585984f3e4120366John McCall if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID)) 2387a45680b7e7c49ea9893c6cff585984f3e4120366John McCall return emitLibraryCall(*this, FD, E, EmitScalarExpr(E->getCallee())); 23881eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 238987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Check that a call to a target specific builtin has the correct target 239087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // features. 239187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // This is down here to avoid non-target specific builtins, however, if 239287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // generic builtins start to require generic target features then we 239387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // can move this up to the beginning of the function. 239487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar checkTargetFeatures(E, FD); 239587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 2396b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner // See if we have a target specific intrinsic. 239787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar const char *Name = getContext().BuiltinInfo.getName(BuiltinID); 239855cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic; 239955cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar if (const char *Prefix = 2400c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch())) { 240155cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar IntrinsicID = Intrinsic::getIntrinsicForGCCBuiltin(Prefix, Name); 2402c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines // NOTE we dont need to perform a compatibility flag check here since the 2403c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines // intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the 2404c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines // MS builtins via ALL_MS_LANGUAGES and are filtered earlier. 2405c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines if (IntrinsicID == Intrinsic::not_intrinsic) 2406c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix, Name); 2407c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines } 24081eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2409b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner if (IntrinsicID != Intrinsic::not_intrinsic) { 2410b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner SmallVector<Value*, 16> Args; 24111eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 241246c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner // Find out if any arguments are required to be integer constant 241346c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner // expressions. 241446c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner unsigned ICEArguments = 0; 241546c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner ASTContext::GetBuiltinTypeError Error; 241646c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); 241746c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner assert(Error == ASTContext::GE_None && "Should not codegen an error"); 241846c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner 2419b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner Function *F = CGM.getIntrinsic(IntrinsicID); 24202acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::FunctionType *FTy = F->getFunctionType(); 24211eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2422b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) { 242346c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner Value *ArgValue; 242446c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner // If this is a normal argument, just emit it as a scalar. 242546c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner if ((ICEArguments & (1 << i)) == 0) { 242646c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner ArgValue = EmitScalarExpr(E->getArg(i)); 242746c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner } else { 2428258f930227c1a102c9c22eee88df65f748863425Jim Grosbach // If this is required to be a constant, constant fold it so that we 242946c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner // know that the generated intrinsic gets a ConstantInt. 243046c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner llvm::APSInt Result; 243146c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result,getContext()); 243246c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner assert(IsConst && "Constant arg isn't actually constant?"); 243346c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner (void)IsConst; 2434d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall ArgValue = llvm::ConstantInt::get(getLLVMContext(), Result); 243546c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner } 24361eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2437b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner // If the intrinsic arg type is different from the builtin arg type 2438b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner // we need to do a bit cast. 24392acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *PTy = FTy->getParamType(i); 2440b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner if (PTy != ArgValue->getType()) { 2441b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner assert(PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) && 2442b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner "Must be able to losslessly bit cast to param"); 2443b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner ArgValue = Builder.CreateBitCast(ArgValue, PTy); 2444b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner } 24451eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2446b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner Args.push_back(ArgValue); 2447b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner } 24481eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 24494c7d9f1507d0f102bd4133bba63348636facd469Jay Foad Value *V = Builder.CreateCall(F, Args); 2450b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner QualType BuiltinRetType = E->getType(); 24511eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 24528b418685e9e4f02f4eb2a76e1ec063e07552b68dChris Lattner llvm::Type *RetTy = VoidTy; 2453258f930227c1a102c9c22eee88df65f748863425Jim Grosbach if (!BuiltinRetType->isVoidType()) 24548b418685e9e4f02f4eb2a76e1ec063e07552b68dChris Lattner RetTy = ConvertType(BuiltinRetType); 24551eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2456b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner if (RetTy != V->getType()) { 2457b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner assert(V->getType()->canLosslesslyBitCastTo(RetTy) && 2458b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner "Must be able to losslessly bit cast result type"); 2459b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner V = Builder.CreateBitCast(V, RetTy); 2460b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner } 24611eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2462b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner return RValue::get(V); 2463b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner } 24641eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2465b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner // See if we have a target specific builtin that needs to be lowered. 2466f02e9ddf5efc75917af712b3c7f909581205f0a5Daniel Dunbar if (Value *V = EmitTargetBuiltinExpr(BuiltinID, E)) 2467b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner return RValue::get(V); 24681eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2469488e993a135ce700b982bf099c3d6b856301d642Daniel Dunbar ErrorUnsupported(E, "builtin function"); 24701eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2471b7cfe88e88cb4f46308de89cf3f0c81bfe624128Chris Lattner // Unknown builtin, for now just dump it out and return undef. 24729d232c884ea9872d6555df0fd7359699819bc1f1John McCall return GetUndefRValue(E->getType()); 24731eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump} 2474564f1de67d7ba43646b8740db86d6269e3dfbe0bAnders Carlsson 247587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic Value *EmitTargetArchBuiltinExpr(CodeGenFunction *CGF, 247687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar unsigned BuiltinID, const CallExpr *E, 247787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Triple::ArchType Arch) { 247887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar switch (Arch) { 24792752c0137d95aa2f4ee1cdff4b564bac842e041bChris Lattner case llvm::Triple::arm: 2480651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case llvm::Triple::armeb: 24812752c0137d95aa2f4ee1cdff4b564bac842e041bChris Lattner case llvm::Triple::thumb: 2482651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case llvm::Triple::thumbeb: 248387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return CGF->EmitARMBuiltinExpr(BuiltinID, E); 24846bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case llvm::Triple::aarch64: 24856bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case llvm::Triple::aarch64_be: 248687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return CGF->EmitAArch64BuiltinExpr(BuiltinID, E); 248755cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar case llvm::Triple::x86: 248855cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar case llvm::Triple::x86_64: 248987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return CGF->EmitX86BuiltinExpr(BuiltinID, E); 249055cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar case llvm::Triple::ppc: 249155cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar case llvm::Triple::ppc64: 2492ea7fb0ce25acc04664a2e7c2b24af03cef2c0d1fBill Schmidt case llvm::Triple::ppc64le: 249387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return CGF->EmitPPCBuiltinExpr(BuiltinID, E); 2494c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case llvm::Triple::r600: 24950e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case llvm::Triple::amdgcn: 249687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E); 249758878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar case llvm::Triple::systemz: 249887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return CGF->EmitSystemZBuiltinExpr(BuiltinID, E); 249987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case llvm::Triple::nvptx: 250087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case llvm::Triple::nvptx64: 250187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return CGF->EmitNVPTXBuiltinExpr(BuiltinID, E); 250287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case llvm::Triple::wasm32: 250387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case llvm::Triple::wasm64: 250487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return CGF->EmitWebAssemblyBuiltinExpr(BuiltinID, E); 250555cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar default: 25066bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 250755cc2ed722e041228670d26d548e5590e355acedDaniel Dunbar } 2508f02e9ddf5efc75917af712b3c7f909581205f0a5Daniel Dunbar} 2509f02e9ddf5efc75917af712b3c7f909581205f0a5Daniel Dunbar 251087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga NainarValue *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID, 251187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar const CallExpr *E) { 251287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar if (getContext().BuiltinInfo.isAuxBuiltinID(BuiltinID)) { 251387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar assert(getContext().getAuxTargetInfo() && "Missing aux target info"); 251487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return EmitTargetArchBuiltinExpr( 251587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar this, getContext().BuiltinInfo.getAuxBuiltinID(BuiltinID), E, 251687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar getContext().getAuxTargetInfo()->getTriple().getArch()); 251787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 251887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 251987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return EmitTargetArchBuiltinExpr(this, BuiltinID, E, 252087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar getTarget().getTriple().getArch()); 252187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 252287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 25238b418685e9e4f02f4eb2a76e1ec063e07552b68dChris Lattnerstatic llvm::VectorType *GetNeonType(CodeGenFunction *CGF, 2524aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu NeonTypeFlags TypeFlags, 2525aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu bool V1Ty=false) { 252683084c863572b48579767a4dd5dc686e1a8d669dNAKAMURA Takumi int IsQuad = TypeFlags.isQuad(); 252783084c863572b48579767a4dd5dc686e1a8d669dNAKAMURA Takumi switch (TypeFlags.getEltType()) { 2528da95f73b59f9af964e33725c515139d34c90c863Bob Wilson case NeonTypeFlags::Int8: 2529da95f73b59f9af964e33725c515139d34c90c863Bob Wilson case NeonTypeFlags::Poly8: 2530aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu return llvm::VectorType::get(CGF->Int8Ty, V1Ty ? 1 : (8 << IsQuad)); 2531da95f73b59f9af964e33725c515139d34c90c863Bob Wilson case NeonTypeFlags::Int16: 2532da95f73b59f9af964e33725c515139d34c90c863Bob Wilson case NeonTypeFlags::Poly16: 2533da95f73b59f9af964e33725c515139d34c90c863Bob Wilson case NeonTypeFlags::Float16: 2534aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu return llvm::VectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad)); 2535da95f73b59f9af964e33725c515139d34c90c863Bob Wilson case NeonTypeFlags::Int32: 2536aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu return llvm::VectorType::get(CGF->Int32Ty, V1Ty ? 1 : (2 << IsQuad)); 2537da95f73b59f9af964e33725c515139d34c90c863Bob Wilson case NeonTypeFlags::Int64: 2538624bb5e59dbcc24efeee7dff12c9b48d2b5077e9Kevin Qin case NeonTypeFlags::Poly64: 2539aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu return llvm::VectorType::get(CGF->Int64Ty, V1Ty ? 1 : (1 << IsQuad)); 2540651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NeonTypeFlags::Poly128: 2541651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: i128 and f128 doesn't get fully support in Clang and llvm. 2542651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // There is a lot of i128 and f128 API missing. 2543651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // so we use v16i8 to represent poly128 and get pattern matched. 2544651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return llvm::VectorType::get(CGF->Int8Ty, 16); 2545da95f73b59f9af964e33725c515139d34c90c863Bob Wilson case NeonTypeFlags::Float32: 2546aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu return llvm::VectorType::get(CGF->FloatTy, V1Ty ? 1 : (2 << IsQuad)); 2547b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover case NeonTypeFlags::Float64: 2548aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu return llvm::VectorType::get(CGF->DoubleTy, V1Ty ? 1 : (1 << IsQuad)); 2549561d3abc881033776ece385a01a510e1cbc1fa92David Blaikie } 25500cd6bd62f3f7ee42f08ad130395ac65564768990Benjamin Kramer llvm_unreachable("Unknown vector element type!"); 2551998622c10198a25ba06c93d7e908f88ba0acc920Nate Begeman} 2552998622c10198a25ba06c93d7e908f88ba0acc920Nate Begeman 255387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic llvm::VectorType *GetFloatNeonType(CodeGenFunction *CGF, 255487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar NeonTypeFlags IntTypeFlags) { 255587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar int IsQuad = IntTypeFlags.isQuad(); 255687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar switch (IntTypeFlags.getEltType()) { 255787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NeonTypeFlags::Int32: 255887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return llvm::VectorType::get(CGF->FloatTy, (2 << IsQuad)); 255987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NeonTypeFlags::Int64: 256087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return llvm::VectorType::get(CGF->DoubleTy, (1 << IsQuad)); 256187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar default: 256287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm_unreachable("Type can't be converted to floating-point!"); 256387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 256487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 256587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 2566cf55652cf668c1402eee0b12edd2e5a1bc34d7a1Bob WilsonValue *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C) { 25674967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar unsigned nElts = V->getType()->getVectorNumElements(); 25682ce8842641cc312628c4be836d34eb250e7c3f78Chris Lattner Value* SV = llvm::ConstantVector::getSplat(nElts, C); 2569d075c01c359b9cc120c3accc7166990f9f4ac423Nate Begeman return Builder.CreateShuffleVector(V, V, SV, "lane"); 2570d075c01c359b9cc120c3accc7166990f9f4ac423Nate Begeman} 2571d075c01c359b9cc120c3accc7166990f9f4ac423Nate Begeman 257230d91718a676177f0d0d0210ce4fdb4f616df6e5Nate BegemanValue *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops, 2573db3d4d036037f379f12643e067b229862d61e932Bob Wilson const char *name, 257461eecf5aea018ea65c9ab0bccacd2996b15c632dNate Begeman unsigned shift, bool rightshift) { 257530d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman unsigned j = 0; 257630d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end(); 257730d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman ai != ae; ++ai, ++j) 257861eecf5aea018ea65c9ab0bccacd2996b15c632dNate Begeman if (shift > 0 && shift == j) 257961eecf5aea018ea65c9ab0bccacd2996b15c632dNate Begeman Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift); 258061eecf5aea018ea65c9ab0bccacd2996b15c632dNate Begeman else 258161eecf5aea018ea65c9ab0bccacd2996b15c632dNate Begeman Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name); 258230d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman 25834c7d9f1507d0f102bd4133bba63348636facd469Jay Foad return Builder.CreateCall(F, Ops, name); 258430d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman} 258530d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman 2586258f930227c1a102c9c22eee88df65f748863425Jim GrosbachValue *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty, 2587464ccb68f22a7e1c0a2844551c16f721540c91c3Nate Begeman bool neg) { 25882ce8842641cc312628c4be836d34eb250e7c3f78Chris Lattner int SV = cast<ConstantInt>(V)->getSExtValue(); 258987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return ConstantInt::get(Ty, neg ? -SV : SV); 2590464ccb68f22a7e1c0a2844551c16f721540c91c3Nate Begeman} 2591464ccb68f22a7e1c0a2844551c16f721540c91c3Nate Begeman 25927f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville// \brief Right-shift a vector by a constant. 25937f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la VieuvilleValue *CodeGenFunction::EmitNeonRShiftImm(Value *Vec, Value *Shift, 25947f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville llvm::Type *Ty, bool usgn, 25957f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville const char *name) { 25967f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville llvm::VectorType *VTy = cast<llvm::VectorType>(Ty); 25977f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville 25987f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville int ShiftAmt = cast<ConstantInt>(Shift)->getSExtValue(); 25997f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville int EltSize = VTy->getScalarSizeInBits(); 26007f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville 26017f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville Vec = Builder.CreateBitCast(Vec, Ty); 26027f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville 26037f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville // lshr/ashr are undefined when the shift amount is equal to the vector 26047f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville // element size. 26057f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville if (ShiftAmt == EltSize) { 26067f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville if (usgn) { 26077f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville // Right-shifting an unsigned value by its size yields 0. 260887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return llvm::ConstantAggregateZero::get(VTy); 26097f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville } else { 26107f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville // Right-shifting a signed value by its size is equivalent 26117f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville // to a shift of size-1. 26127f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville --ShiftAmt; 26137f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt); 26147f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville } 26157f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville } 26167f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville 26177f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville Shift = EmitNeonShiftVector(Shift, Ty, false); 26187f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville if (usgn) 26197f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville return Builder.CreateLShr(Vec, Shift, name); 26207f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville else 26217f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville return Builder.CreateAShr(Vec, Shift, name); 26227f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville} 26237f0ff70c391afeb7811b7ddcb06bfbbd5c6cf8f9Amaury de la Vieuville 2624651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesenum { 2625651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AddRetType = (1 << 0), 2626651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Add1ArgType = (1 << 1), 2627651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Add2ArgTypes = (1 << 2), 2628651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2629651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VectorizeRetType = (1 << 3), 2630651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VectorizeArgTypes = (1 << 4), 2631651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2632651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines InventFloatType = (1 << 5), 2633651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines UnsignedAlts = (1 << 6), 2634651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2635651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Use64BitVectors = (1 << 7), 2636651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Use128BitVectors = (1 << 8), 2637651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2638651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Vectorize1ArgType = Add1ArgType | VectorizeArgTypes, 2639651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VectorRet = AddRetType | VectorizeRetType, 2640651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VectorRetGetArgs01 = 2641651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AddRetType | Add2ArgTypes | VectorizeRetType | VectorizeArgTypes, 2642651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines FpCmpzModifiers = 2643651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AddRetType | VectorizeRetType | Add1ArgType | InventFloatType 2644651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines}; 2645651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 264687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarnamespace { 264787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstruct NeonIntrinsicInfo { 264887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar const char *NameHint; 2649651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned BuiltinID; 2650651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned LLVMIntrinsic; 2651651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned AltLLVMIntrinsic; 2652651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned TypeModifier; 2653651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2654651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool operator<(unsigned RHSBuiltinID) const { 2655651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return BuiltinID < RHSBuiltinID; 2656651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 265787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar bool operator<(const NeonIntrinsicInfo &TE) const { 265887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return BuiltinID < TE.BuiltinID; 265987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 2660651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines}; 266187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} // end anonymous namespace 2662651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2663651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#define NEONMAP0(NameBase) \ 266487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar { #NameBase, NEON::BI__builtin_neon_ ## NameBase, 0, 0, 0 } 2665651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2666651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \ 266787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \ 266887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Intrinsic::LLVMIntrinsic, 0, TypeModifier } 2669651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2670651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \ 267187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \ 2672651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \ 267387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar TypeModifier } 2674651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 267587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic const NeonIntrinsicInfo ARMSIMDIntrinsicMap [] = { 2676651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vabd_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts), 2677651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vabdq_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts), 2678651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vabs_v, arm_neon_vabs, 0), 2679651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vabsq_v, arm_neon_vabs, 0), 2680651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vaddhn_v), 2681651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vaesdq_v, arm_neon_aesd, 0), 2682651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vaeseq_v, arm_neon_aese, 0), 2683651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vaesimcq_v, arm_neon_aesimc, 0), 2684651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vaesmcq_v, arm_neon_aesmc, 0), 2685651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vbsl_v, arm_neon_vbsl, AddRetType), 2686651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vbslq_v, arm_neon_vbsl, AddRetType), 2687651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcage_v, arm_neon_vacge, 0), 2688651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcageq_v, arm_neon_vacge, 0), 2689651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcagt_v, arm_neon_vacgt, 0), 2690651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcagtq_v, arm_neon_vacgt, 0), 2691651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcale_v, arm_neon_vacge, 0), 2692651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcaleq_v, arm_neon_vacge, 0), 2693651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcalt_v, arm_neon_vacgt, 0), 2694651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcaltq_v, arm_neon_vacgt, 0), 2695651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcls_v, arm_neon_vcls, Add1ArgType), 2696651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vclsq_v, arm_neon_vcls, Add1ArgType), 2697651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vclz_v, ctlz, Add1ArgType), 2698651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vclzq_v, ctlz, Add1ArgType), 2699651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcnt_v, ctpop, Add1ArgType), 2700651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcntq_v, ctpop, Add1ArgType), 270187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0), 2702651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0), 2703651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvt_f32_v), 2704651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0), 2705651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0), 2706651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0), 2707651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0), 2708651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0), 2709651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvt_s32_v), 2710651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvt_s64_v), 2711651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvt_u32_v), 2712651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvt_u64_v), 2713651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0), 2714651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0), 2715651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0), 2716651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0), 2717651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0), 2718651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0), 2719651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0), 2720651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0), 2721651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0), 2722651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0), 2723651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0), 2724651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0), 2725651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0), 2726651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0), 2727651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0), 2728651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0), 2729651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0), 2730651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0), 2731651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0), 2732651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0), 2733651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0), 2734651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0), 2735651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0), 2736651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0), 2737651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0), 2738651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0), 2739651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0), 2740651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0), 2741651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0), 2742651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0), 2743651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0), 2744651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0), 2745651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvtq_f32_v), 2746651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0), 2747651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0), 2748651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0), 2749651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0), 2750651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0), 2751651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvtq_s32_v), 2752651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvtq_s64_v), 2753651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvtq_u32_v), 2754651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvtq_u64_v), 2755651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vext_v), 2756651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vextq_v), 2757651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vfma_v), 2758651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vfmaq_v), 2759651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vhadd_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts), 2760651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vhaddq_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts), 2761651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vhsub_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts), 2762651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vhsubq_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts), 2763651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vld1_dup_v), 2764651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld1_v, arm_neon_vld1, 0), 2765651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vld1q_dup_v), 2766651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld1q_v, arm_neon_vld1, 0), 2767651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0), 2768651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld2_v, arm_neon_vld2, 0), 2769651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0), 2770651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld2q_v, arm_neon_vld2, 0), 2771651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0), 2772651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld3_v, arm_neon_vld3, 0), 2773651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0), 2774651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld3q_v, arm_neon_vld3, 0), 2775651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0), 2776651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld4_v, arm_neon_vld4, 0), 2777651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0), 2778651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vld4q_v, arm_neon_vld4, 0), 2779651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vmax_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts), 2780176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vmaxnm_v, arm_neon_vmaxnm, Add1ArgType), 2781176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vmaxnmq_v, arm_neon_vmaxnm, Add1ArgType), 2782651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vmaxq_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts), 2783651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vmin_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts), 2784176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vminnm_v, arm_neon_vminnm, Add1ArgType), 2785176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vminnmq_v, arm_neon_vminnm, Add1ArgType), 2786651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vminq_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts), 2787651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vmovl_v), 2788651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vmovn_v), 2789651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vmul_v, arm_neon_vmulp, Add1ArgType), 2790651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vmull_v), 2791651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vmulq_v, arm_neon_vmulp, Add1ArgType), 2792651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vpadal_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts), 2793651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vpadalq_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts), 2794651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vpadd_v, arm_neon_vpadd, Add1ArgType), 2795651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vpaddl_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts), 2796651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vpaddlq_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts), 2797651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vpaddq_v, arm_neon_vpadd, Add1ArgType), 2798651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vpmax_v, arm_neon_vpmaxu, arm_neon_vpmaxs, Add1ArgType | UnsignedAlts), 2799651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vpmin_v, arm_neon_vpminu, arm_neon_vpmins, Add1ArgType | UnsignedAlts), 2800651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqabs_v, arm_neon_vqabs, Add1ArgType), 2801651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqabsq_v, arm_neon_vqabs, Add1ArgType), 2802651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqadd_v, arm_neon_vqaddu, arm_neon_vqadds, Add1ArgType | UnsignedAlts), 2803651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqaddq_v, arm_neon_vqaddu, arm_neon_vqadds, Add1ArgType | UnsignedAlts), 2804651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqdmlal_v, arm_neon_vqdmull, arm_neon_vqadds, 0), 2805651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, arm_neon_vqsubs, 0), 2806651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqdmulh_v, arm_neon_vqdmulh, Add1ArgType), 2807651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqdmulhq_v, arm_neon_vqdmulh, Add1ArgType), 2808651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqdmull_v, arm_neon_vqdmull, Add1ArgType), 2809651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqmovn_v, arm_neon_vqmovnu, arm_neon_vqmovns, Add1ArgType | UnsignedAlts), 2810651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqmovun_v, arm_neon_vqmovnsu, Add1ArgType), 2811651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqneg_v, arm_neon_vqneg, Add1ArgType), 2812651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqnegq_v, arm_neon_vqneg, Add1ArgType), 2813651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqrdmulh_v, arm_neon_vqrdmulh, Add1ArgType), 2814651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vqrdmulhq_v, arm_neon_vqrdmulh, Add1ArgType), 2815651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqrshl_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts), 2816651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqrshlq_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts), 2817651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqshl_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts), 2818651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqshl_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts), 2819651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqshlq_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts), 2820651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqshlq_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts), 2821176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0), 2822176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0), 2823651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqsub_v, arm_neon_vqsubu, arm_neon_vqsubs, Add1ArgType | UnsignedAlts), 2824651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vqsubq_v, arm_neon_vqsubu, arm_neon_vqsubs, Add1ArgType | UnsignedAlts), 2825651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vraddhn_v, arm_neon_vraddhn, Add1ArgType), 2826651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0), 2827651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0), 2828651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vrecps_v, arm_neon_vrecps, Add1ArgType), 2829651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vrecpsq_v, arm_neon_vrecps, Add1ArgType), 2830651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vrhadd_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts), 2831651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vrhaddq_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts), 2832176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vrnd_v, arm_neon_vrintz, Add1ArgType), 2833176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vrnda_v, arm_neon_vrinta, Add1ArgType), 2834176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vrndaq_v, arm_neon_vrinta, Add1ArgType), 2835176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vrndm_v, arm_neon_vrintm, Add1ArgType), 2836176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vrndmq_v, arm_neon_vrintm, Add1ArgType), 2837176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vrndn_v, arm_neon_vrintn, Add1ArgType), 2838176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vrndnq_v, arm_neon_vrintn, Add1ArgType), 2839176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vrndp_v, arm_neon_vrintp, Add1ArgType), 2840176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vrndpq_v, arm_neon_vrintp, Add1ArgType), 2841176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vrndq_v, arm_neon_vrintz, Add1ArgType), 2842176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vrndx_v, arm_neon_vrintx, Add1ArgType), 2843176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vrndxq_v, arm_neon_vrintx, Add1ArgType), 2844651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vrshl_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts), 2845651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vrshlq_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts), 2846176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP2(vrshr_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts), 2847176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP2(vrshrq_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts), 2848651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0), 2849651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0), 2850651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vrsqrts_v, arm_neon_vrsqrts, Add1ArgType), 2851651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vrsqrtsq_v, arm_neon_vrsqrts, Add1ArgType), 2852651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vrsubhn_v, arm_neon_vrsubhn, Add1ArgType), 2853651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vsha1su0q_v, arm_neon_sha1su0, 0), 2854651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vsha1su1q_v, arm_neon_sha1su1, 0), 2855651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vsha256h2q_v, arm_neon_sha256h2, 0), 2856651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vsha256hq_v, arm_neon_sha256h, 0), 2857651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vsha256su0q_v, arm_neon_sha256su0, 0), 2858651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vsha256su1q_v, arm_neon_sha256su1, 0), 2859651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshl_n_v), 2860651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vshl_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts), 2861651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshll_n_v), 2862651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshlq_n_v), 2863651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP2(vshlq_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts), 2864651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshr_n_v), 2865651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshrn_n_v), 2866651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshrq_n_v), 2867651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst1_v, arm_neon_vst1, 0), 2868651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst1q_v, arm_neon_vst1, 0), 2869651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0), 2870651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst2_v, arm_neon_vst2, 0), 2871651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0), 2872651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst2q_v, arm_neon_vst2, 0), 2873651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0), 2874651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst3_v, arm_neon_vst3, 0), 2875651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0), 2876651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst3q_v, arm_neon_vst3, 0), 2877651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0), 2878651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst4_v, arm_neon_vst4, 0), 2879651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0), 2880651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vst4q_v, arm_neon_vst4, 0), 2881651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vsubhn_v), 2882651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vtrn_v), 2883651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vtrnq_v), 2884651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vtst_v), 2885651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vtstq_v), 2886651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vuzp_v), 2887651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vuzpq_v), 2888651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vzip_v), 2889651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vzipq_v) 2890651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines}; 2891651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 289287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic const NeonIntrinsicInfo AArch64SIMDIntrinsicMap[] = { 28936bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vabs_v, aarch64_neon_abs, 0), 28946bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vabsq_v, aarch64_neon_abs, 0), 2895651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vaddhn_v), 28966bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaesdq_v, aarch64_crypto_aesd, 0), 28976bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaeseq_v, aarch64_crypto_aese, 0), 28986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaesimcq_v, aarch64_crypto_aesimc, 0), 28996bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaesmcq_v, aarch64_crypto_aesmc, 0), 29006bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcage_v, aarch64_neon_facge, 0), 29016bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcageq_v, aarch64_neon_facge, 0), 29026bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcagt_v, aarch64_neon_facgt, 0), 29036bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0), 29046bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcale_v, aarch64_neon_facge, 0), 29056bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcaleq_v, aarch64_neon_facge, 0), 29066bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcalt_v, aarch64_neon_facgt, 0), 29076bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0), 29086bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcls_v, aarch64_neon_cls, Add1ArgType), 29096bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vclsq_v, aarch64_neon_cls, Add1ArgType), 2910651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vclz_v, ctlz, Add1ArgType), 2911651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vclzq_v, ctlz, Add1ArgType), 2912651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcnt_v, ctpop, Add1ArgType), 2913651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP1(vcntq_v, ctpop, Add1ArgType), 291487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0), 29156bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0), 2916651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvt_f32_v), 29176bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 29186bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 29196bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0), 29206bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0), 29216bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0), 29226bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0), 2923651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vcvtq_f32_v), 29246bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 29256bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 29266bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0), 29276bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0), 29286bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0), 29296bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0), 29306bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtx_f32_v, aarch64_neon_fcvtxn, AddRetType | Add1ArgType), 2931651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vext_v), 2932651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vextq_v), 2933651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vfma_v), 2934651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vfmaq_v), 29356bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vhadd_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts), 29366bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vhaddq_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts), 29376bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vhsub_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts), 29386bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vhsubq_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts), 2939651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vmovl_v), 2940651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vmovn_v), 29416bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmul_v, aarch64_neon_pmul, Add1ArgType), 29426bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmulq_v, aarch64_neon_pmul, Add1ArgType), 29436bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpadd_v, aarch64_neon_addp, Add1ArgType), 29446bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vpaddl_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts), 29456bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vpaddlq_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts), 29466bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpaddq_v, aarch64_neon_addp, Add1ArgType), 29476bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqabs_v, aarch64_neon_sqabs, Add1ArgType), 29486bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqabsq_v, aarch64_neon_sqabs, Add1ArgType), 29496bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqadd_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts), 29506bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqaddq_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts), 29516bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0), 29526bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0), 29536bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqdmulh_v, aarch64_neon_sqdmulh, Add1ArgType), 29546bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqdmulhq_v, aarch64_neon_sqdmulh, Add1ArgType), 29556bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqdmull_v, aarch64_neon_sqdmull, Add1ArgType), 29566bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqmovn_v, aarch64_neon_uqxtn, aarch64_neon_sqxtn, Add1ArgType | UnsignedAlts), 29576bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovun_v, aarch64_neon_sqxtun, Add1ArgType), 29586bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqneg_v, aarch64_neon_sqneg, Add1ArgType), 29596bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqnegq_v, aarch64_neon_sqneg, Add1ArgType), 29606bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrdmulh_v, aarch64_neon_sqrdmulh, Add1ArgType), 29616bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrdmulhq_v, aarch64_neon_sqrdmulh, Add1ArgType), 29626bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqrshl_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts), 29636bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqrshlq_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts), 29646bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqshl_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl, UnsignedAlts), 29656bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqshl_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts), 29666bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqshlq_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl,UnsignedAlts), 29676bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqshlq_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts), 2968176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0), 2969176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0), 29706bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqsub_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts), 29716bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vqsubq_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts), 29726bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vraddhn_v, aarch64_neon_raddhn, Add1ArgType), 29736bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0), 29746bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0), 29756bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrecps_v, aarch64_neon_frecps, Add1ArgType), 29766bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrecpsq_v, aarch64_neon_frecps, Add1ArgType), 29776bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vrhadd_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts), 29786bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vrhaddq_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts), 29796bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vrshl_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts), 29806bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vrshlq_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts), 2981176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP2(vrshr_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts), 2982176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines NEONMAP2(vrshrq_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts), 29836bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0), 29846bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0), 29856bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrsqrts_v, aarch64_neon_frsqrts, Add1ArgType), 29866bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrsqrtsq_v, aarch64_neon_frsqrts, Add1ArgType), 29876bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrsubhn_v, aarch64_neon_rsubhn, Add1ArgType), 29886bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha1su0q_v, aarch64_crypto_sha1su0, 0), 29896bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha1su1q_v, aarch64_crypto_sha1su1, 0), 29906bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha256h2q_v, aarch64_crypto_sha256h2, 0), 29916bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha256hq_v, aarch64_crypto_sha256h, 0), 29926bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha256su0q_v, aarch64_crypto_sha256su0, 0), 29936bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha256su1q_v, aarch64_crypto_sha256su1, 0), 2994651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshl_n_v), 29956bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vshl_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts), 2996651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshll_n_v), 2997651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshlq_n_v), 29986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP2(vshlq_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts), 2999651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshr_n_v), 3000651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshrn_n_v), 3001651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vshrq_n_v), 3002651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vsubhn_v), 3003651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vtst_v), 3004651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NEONMAP0(vtstq_v), 3005651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines}; 3006651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 300787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic const NeonIntrinsicInfo AArch64SISDIntrinsicMap[] = { 30086bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vabdd_f64, aarch64_sisd_fabd, Add1ArgType), 30096bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vabds_f32, aarch64_sisd_fabd, Add1ArgType), 30106bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vabsd_s64, aarch64_neon_abs, Add1ArgType), 30116bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddlv_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType), 30126bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddlv_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType), 30136bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddlvq_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType), 30146bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddlvq_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType), 30156bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddv_f32, aarch64_neon_faddv, AddRetType | Add1ArgType), 30166bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddv_s32, aarch64_neon_saddv, AddRetType | Add1ArgType), 30176bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddv_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType), 30186bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddvq_f32, aarch64_neon_faddv, AddRetType | Add1ArgType), 30196bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddvq_f64, aarch64_neon_faddv, AddRetType | Add1ArgType), 30206bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddvq_s32, aarch64_neon_saddv, AddRetType | Add1ArgType), 30216bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddvq_s64, aarch64_neon_saddv, AddRetType | Add1ArgType), 30226bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddvq_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType), 30236bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vaddvq_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType), 30246bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcaged_f64, aarch64_neon_facge, AddRetType | Add1ArgType), 30256bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcages_f32, aarch64_neon_facge, AddRetType | Add1ArgType), 30266bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcagtd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType), 30276bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcagts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType), 30286bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcaled_f64, aarch64_neon_facge, AddRetType | Add1ArgType), 30296bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcales_f32, aarch64_neon_facge, AddRetType | Add1ArgType), 30306bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcaltd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType), 30316bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcalts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType), 30326bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtad_s64_f64, aarch64_neon_fcvtas, AddRetType | Add1ArgType), 30336bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtad_u64_f64, aarch64_neon_fcvtau, AddRetType | Add1ArgType), 30346bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtas_s32_f32, aarch64_neon_fcvtas, AddRetType | Add1ArgType), 30356bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtas_u32_f32, aarch64_neon_fcvtau, AddRetType | Add1ArgType), 30366bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtd_n_f64_s64, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType), 30376bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtd_n_f64_u64, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType), 30386bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtd_n_s64_f64, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType), 30396bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtd_n_u64_f64, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType), 30406bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtmd_s64_f64, aarch64_neon_fcvtms, AddRetType | Add1ArgType), 30416bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtmd_u64_f64, aarch64_neon_fcvtmu, AddRetType | Add1ArgType), 30426bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtms_s32_f32, aarch64_neon_fcvtms, AddRetType | Add1ArgType), 30436bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtms_u32_f32, aarch64_neon_fcvtmu, AddRetType | Add1ArgType), 30446bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtnd_s64_f64, aarch64_neon_fcvtns, AddRetType | Add1ArgType), 30456bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtnd_u64_f64, aarch64_neon_fcvtnu, AddRetType | Add1ArgType), 30466bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtns_s32_f32, aarch64_neon_fcvtns, AddRetType | Add1ArgType), 30476bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtns_u32_f32, aarch64_neon_fcvtnu, AddRetType | Add1ArgType), 30486bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtpd_s64_f64, aarch64_neon_fcvtps, AddRetType | Add1ArgType), 30496bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtpd_u64_f64, aarch64_neon_fcvtpu, AddRetType | Add1ArgType), 30506bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtps_s32_f32, aarch64_neon_fcvtps, AddRetType | Add1ArgType), 30516bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtps_u32_f32, aarch64_neon_fcvtpu, AddRetType | Add1ArgType), 30526bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvts_n_f32_s32, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType), 30536bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvts_n_f32_u32, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType), 30546bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvts_n_s32_f32, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType), 30556bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvts_n_u32_f32, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType), 30566bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0), 30576bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxnmv_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 30586bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxnmvq_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 30596bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxnmvq_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 30606bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxv_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 30616bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxv_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType), 30626bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxv_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType), 30636bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxvq_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 30646bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxvq_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 30656bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxvq_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType), 30666bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmaxvq_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType), 30676bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminnmv_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 30686bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminnmvq_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 30696bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminnmvq_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 30706bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminv_f32, aarch64_neon_fminv, AddRetType | Add1ArgType), 30716bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminv_s32, aarch64_neon_sminv, AddRetType | Add1ArgType), 30726bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminv_u32, aarch64_neon_uminv, AddRetType | Add1ArgType), 30736bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminvq_f32, aarch64_neon_fminv, AddRetType | Add1ArgType), 30746bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminvq_f64, aarch64_neon_fminv, AddRetType | Add1ArgType), 30756bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminvq_s32, aarch64_neon_sminv, AddRetType | Add1ArgType), 30766bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vminvq_u32, aarch64_neon_uminv, AddRetType | Add1ArgType), 30776bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0), 30786bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmulxd_f64, aarch64_neon_fmulx, Add1ArgType), 30796bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vmulxs_f32, aarch64_neon_fmulx, Add1ArgType), 30806bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpaddd_s64, aarch64_neon_uaddv, AddRetType | Add1ArgType), 30816bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpaddd_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType), 30826bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpmaxnmqd_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 30836bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpmaxnms_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 30846bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpmaxqd_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 30856bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpmaxs_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 30866bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpminnmqd_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 30876bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpminnms_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 30886bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpminqd_f64, aarch64_neon_fminv, AddRetType | Add1ArgType), 30896bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vpmins_f32, aarch64_neon_fminv, AddRetType | Add1ArgType), 30906bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqabsb_s8, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors), 30916bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqabsd_s64, aarch64_neon_sqabs, Add1ArgType), 30926bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqabsh_s16, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors), 30936bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqabss_s32, aarch64_neon_sqabs, Add1ArgType), 30946bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqaddb_s8, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors), 30956bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqaddb_u8, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors), 30966bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqaddd_s64, aarch64_neon_sqadd, Add1ArgType), 30976bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqaddd_u64, aarch64_neon_uqadd, Add1ArgType), 30986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqaddh_s16, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors), 30996bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqaddh_u16, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors), 31006bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqadds_s32, aarch64_neon_sqadd, Add1ArgType), 31016bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqadds_u32, aarch64_neon_uqadd, Add1ArgType), 31026bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqdmulhh_s16, aarch64_neon_sqdmulh, Vectorize1ArgType | Use64BitVectors), 31036bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqdmulhs_s32, aarch64_neon_sqdmulh, Add1ArgType), 31046bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqdmullh_s16, aarch64_neon_sqdmull, VectorRet | Use128BitVectors), 31056bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0), 31066bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovnd_s64, aarch64_neon_scalar_sqxtn, AddRetType | Add1ArgType), 31076bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovnd_u64, aarch64_neon_scalar_uqxtn, AddRetType | Add1ArgType), 31086bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovnh_s16, aarch64_neon_sqxtn, VectorRet | Use64BitVectors), 31096bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovnh_u16, aarch64_neon_uqxtn, VectorRet | Use64BitVectors), 31106bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovns_s32, aarch64_neon_sqxtn, VectorRet | Use64BitVectors), 31116bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovns_u32, aarch64_neon_uqxtn, VectorRet | Use64BitVectors), 31126bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovund_s64, aarch64_neon_scalar_sqxtun, AddRetType | Add1ArgType), 31136bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovunh_s16, aarch64_neon_sqxtun, VectorRet | Use64BitVectors), 31146bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqmovuns_s32, aarch64_neon_sqxtun, VectorRet | Use64BitVectors), 31156bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqnegb_s8, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors), 31166bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqnegd_s64, aarch64_neon_sqneg, Add1ArgType), 31176bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqnegh_s16, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors), 31186bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqnegs_s32, aarch64_neon_sqneg, Add1ArgType), 31196bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrdmulhh_s16, aarch64_neon_sqrdmulh, Vectorize1ArgType | Use64BitVectors), 31206bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrdmulhs_s32, aarch64_neon_sqrdmulh, Add1ArgType), 31216bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshlb_s8, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors), 31226bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshlb_u8, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors), 31236bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshld_s64, aarch64_neon_sqrshl, Add1ArgType), 31246bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshld_u64, aarch64_neon_uqrshl, Add1ArgType), 31256bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshlh_s16, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors), 31266bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshlh_u16, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors), 31276bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshls_s32, aarch64_neon_sqrshl, Add1ArgType), 31286bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshls_u32, aarch64_neon_uqrshl, Add1ArgType), 31296bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshrnd_n_s64, aarch64_neon_sqrshrn, AddRetType), 31306bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshrnd_n_u64, aarch64_neon_uqrshrn, AddRetType), 31316bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshrnh_n_s16, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors), 31326bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshrnh_n_u16, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors), 31336bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshrns_n_s32, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors), 31346bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshrns_n_u32, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors), 31356bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshrund_n_s64, aarch64_neon_sqrshrun, AddRetType), 31366bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshrunh_n_s16, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors), 31376bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqrshruns_n_s32, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors), 31386bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlb_n_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 31396bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlb_n_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 31406bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlb_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 31416bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlb_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 31426bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshld_s64, aarch64_neon_sqshl, Add1ArgType), 31436bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshld_u64, aarch64_neon_uqshl, Add1ArgType), 31446bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlh_n_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 31456bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlh_n_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 31466bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlh_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 31476bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlh_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 31486bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshls_n_s32, aarch64_neon_sqshl, Add1ArgType), 31496bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshls_n_u32, aarch64_neon_uqshl, Add1ArgType), 31506bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshls_s32, aarch64_neon_sqshl, Add1ArgType), 31516bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshls_u32, aarch64_neon_uqshl, Add1ArgType), 31526bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlub_n_s8, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors), 31536bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshluh_n_s16, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors), 31546bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshlus_n_s32, aarch64_neon_sqshlu, Add1ArgType), 31556bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshrnd_n_s64, aarch64_neon_sqshrn, AddRetType), 31566bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshrnd_n_u64, aarch64_neon_uqshrn, AddRetType), 31576bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshrnh_n_s16, aarch64_neon_sqshrn, VectorRet | Use64BitVectors), 31586bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshrnh_n_u16, aarch64_neon_uqshrn, VectorRet | Use64BitVectors), 31596bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshrns_n_s32, aarch64_neon_sqshrn, VectorRet | Use64BitVectors), 31606bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshrns_n_u32, aarch64_neon_uqshrn, VectorRet | Use64BitVectors), 31616bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshrund_n_s64, aarch64_neon_sqshrun, AddRetType), 31626bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshrunh_n_s16, aarch64_neon_sqshrun, VectorRet | Use64BitVectors), 31636bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqshruns_n_s32, aarch64_neon_sqshrun, VectorRet | Use64BitVectors), 31646bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqsubb_s8, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors), 31656bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqsubb_u8, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors), 31666bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqsubd_s64, aarch64_neon_sqsub, Add1ArgType), 31676bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqsubd_u64, aarch64_neon_uqsub, Add1ArgType), 31686bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqsubh_s16, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors), 31696bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqsubh_u16, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors), 31706bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqsubs_s32, aarch64_neon_sqsub, Add1ArgType), 31716bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vqsubs_u32, aarch64_neon_uqsub, Add1ArgType), 31726bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrecped_f64, aarch64_neon_frecpe, Add1ArgType), 31736bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrecpes_f32, aarch64_neon_frecpe, Add1ArgType), 31746bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrecpxd_f64, aarch64_neon_frecpx, Add1ArgType), 31756bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrecpxs_f32, aarch64_neon_frecpx, Add1ArgType), 31766bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrshld_s64, aarch64_neon_srshl, Add1ArgType), 31776bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrshld_u64, aarch64_neon_urshl, Add1ArgType), 31786bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrsqrted_f64, aarch64_neon_frsqrte, Add1ArgType), 31796bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrsqrtes_f32, aarch64_neon_frsqrte, Add1ArgType), 31806bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrsqrtsd_f64, aarch64_neon_frsqrts, Add1ArgType), 31816bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vrsqrtss_f32, aarch64_neon_frsqrts, Add1ArgType), 31826bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0), 31836bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0), 31846bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0), 31856bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0), 31866bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vshld_s64, aarch64_neon_sshl, Add1ArgType), 31876bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vshld_u64, aarch64_neon_ushl, Add1ArgType), 31886bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vslid_n_s64, aarch64_neon_vsli, Vectorize1ArgType), 31896bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vslid_n_u64, aarch64_neon_vsli, Vectorize1ArgType), 31906bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsqaddb_u8, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors), 31916bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsqaddd_u64, aarch64_neon_usqadd, Add1ArgType), 31926bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsqaddh_u16, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors), 31936bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsqadds_u32, aarch64_neon_usqadd, Add1ArgType), 31946bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsrid_n_s64, aarch64_neon_vsri, Vectorize1ArgType), 31956bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vsrid_n_u64, aarch64_neon_vsri, Vectorize1ArgType), 31966bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vuqaddb_s8, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors), 31976bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vuqaddd_s64, aarch64_neon_suqadd, Add1ArgType), 31986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vuqaddh_s16, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors), 31996bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines NEONMAP1(vuqadds_s32, aarch64_neon_suqadd, Add1ArgType), 3200651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines}; 3201651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3202651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#undef NEONMAP0 3203651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#undef NEONMAP1 3204651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#undef NEONMAP2 3205651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3206651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic bool NEONSIMDIntrinsicsProvenSorted = false; 3207651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 32086bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hinesstatic bool AArch64SIMDIntrinsicsProvenSorted = false; 32096bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hinesstatic bool AArch64SISDIntrinsicsProvenSorted = false; 3210651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3211651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3212651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic const NeonIntrinsicInfo * 3213c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen HinesfindNeonIntrinsicInMap(ArrayRef<NeonIntrinsicInfo> IntrinsicMap, 3214651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned BuiltinID, bool &MapProvenSorted) { 3215651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3216651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#ifndef NDEBUG 3217651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (!MapProvenSorted) { 321887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar assert(std::is_sorted(std::begin(IntrinsicMap), std::end(IntrinsicMap))); 3219651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines MapProvenSorted = true; 3220651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3221651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#endif 3222651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3223651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const NeonIntrinsicInfo *Builtin = 3224651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines std::lower_bound(IntrinsicMap.begin(), IntrinsicMap.end(), BuiltinID); 3225651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3226651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID) 3227651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builtin; 3228651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 32296bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 3230651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 3231651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3232651f13cea278ec967336033dd032faef0e9fc2ecStephen HinesFunction *CodeGenFunction::LookupNeonLLVMIntrinsic(unsigned IntrinsicID, 3233651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Modifier, 3234651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *ArgType, 3235651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const CallExpr *E) { 3236651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines int VectorSize = 0; 3237651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Modifier & Use64BitVectors) 3238651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VectorSize = 64; 3239651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines else if (Modifier & Use128BitVectors) 3240651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VectorSize = 128; 3241651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3242651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Return type. 3243651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<llvm::Type *, 3> Tys; 3244651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Modifier & AddRetType) { 32450e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext())); 3246651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Modifier & VectorizeRetType) 3247651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::VectorType::get( 3248651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1); 3249651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3250651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Tys.push_back(Ty); 3251651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3252651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3253651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Arguments. 3254651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Modifier & VectorizeArgTypes) { 3255651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1; 3256651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ArgType = llvm::VectorType::get(ArgType, Elts); 3257651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3258651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3259651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Modifier & (Add1ArgType | Add2ArgTypes)) 3260651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Tys.push_back(ArgType); 3261651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3262651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Modifier & Add2ArgTypes) 3263651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Tys.push_back(ArgType); 3264651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3265651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Modifier & InventFloatType) 3266651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Tys.push_back(FloatTy); 3267651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3268651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return CGM.getIntrinsic(IntrinsicID, Tys); 3269651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 3270651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3271651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic Value *EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, 3272651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const NeonIntrinsicInfo &SISDInfo, 3273651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVectorImpl<Value *> &Ops, 3274651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const CallExpr *E) { 3275651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned BuiltinID = SISDInfo.BuiltinID; 3276651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned int Int = SISDInfo.LLVMIntrinsic; 3277651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Modifier = SISDInfo.TypeModifier; 3278651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const char *s = SISDInfo.NameHint; 3279651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3280651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 3281651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_s64: 3282651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_u64: 3283651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcles_f32: 3284651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_f64: 3285651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_s64: 3286651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_u64: 3287651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclts_f32: 3288651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_f64: 3289651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcales_f32: 3290651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcaled_f64: 3291651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcalts_f32: 3292651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcaltd_f64: 3293651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Only one direction of comparisons actually exist, cmle is actually a cmge 3294651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // with swapped operands. The table gives us the right intrinsic but we 3295651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // still need to do the swap. 3296651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines std::swap(Ops[0], Ops[1]); 3297651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 3298651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3299651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3300651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines assert(Int && "Generic code assumes a valid intrinsic"); 3301651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3302651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Determine the type(s) of this overloaded AArch64 intrinsic. 3303651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const Expr *Arg = E->getArg(0); 3304651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *ArgTy = CGF.ConvertType(Arg->getType()); 3305651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGF.LookupNeonLLVMIntrinsic(Int, Modifier, ArgTy, E); 3306651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3307651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines int j = 0; 3308c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines ConstantInt *C0 = ConstantInt::get(CGF.SizeTy, 0); 3309651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end(); 3310651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ai != ae; ++ai, ++j) { 3311651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *ArgTy = ai->getType(); 3312651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Ops[j]->getType()->getPrimitiveSizeInBits() == 3313651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ArgTy->getPrimitiveSizeInBits()) 3314651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines continue; 3315651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3316651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy()); 3317651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The constant argument to an _n_ intrinsic always has Int32Ty, so truncate 3318651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // it before inserting. 3319651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[j] = 3320651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.CreateTruncOrBitCast(Ops[j], ArgTy->getVectorElementType()); 3321651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[j] = 3322651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.CreateInsertElement(UndefValue::get(ArgTy), Ops[j], C0); 3323651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3324651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3325651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Result = CGF.EmitNeonCall(F, Ops, s); 3326651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *ResultType = CGF.ConvertType(E->getType()); 3327651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (ResultType->getPrimitiveSizeInBits() < 3328651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Result->getType()->getPrimitiveSizeInBits()) 3329651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return CGF.Builder.CreateExtractElement(Result, C0); 3330651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3331651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return CGF.Builder.CreateBitCast(Result, ResultType, s); 3332651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 3333651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3334651f13cea278ec967336033dd032faef0e9fc2ecStephen HinesValue *CodeGenFunction::EmitCommonNeonBuiltinExpr( 3335651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic, 3336651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const char *NameHint, unsigned Modifier, const CallExpr *E, 333787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1) { 33382e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu // Get the last argument, which specifies the vector type. 3339651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::APSInt NeonTypeConst; 33402e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu const Expr *Arg = E->getArg(E->getNumArgs() - 1); 3341651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (!Arg->isIntegerConstantExpr(NeonTypeConst, getContext())) 33426bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 33432e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu 33442e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu // Determine the type of this overloaded NEON intrinsic. 3345651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags Type(NeonTypeConst.getZExtValue()); 3346651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Usgn = Type.isUnsigned(); 3347651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Quad = Type.isQuad(); 3348651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3349651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType *VTy = GetNeonType(this, Type); 33502e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu llvm::Type *Ty = VTy; 33512e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu if (!Ty) 33526bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 33532e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu 335487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar auto getAlignmentValue32 = [&](Address addr) -> Value* { 335587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.getInt32(addr.getAlignment().getQuantity()); 335687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar }; 335787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 3358651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Int = LLVMIntrinsic; 3359651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if ((Modifier & UnsignedAlts) && !Usgn) 3360651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = AltLLVMIntrinsic; 33612e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu 33622e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu switch (BuiltinID) { 3363651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines default: break; 3364651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vabs_v: 3365651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vabsq_v: 3366651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (VTy->getElementType()->isFloatingPointTy()) 3367651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs"); 3368651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vabs"); 3369651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddhn_v: { 3370651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType *SrcTy = 3371651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::getExtendedElementVectorType(VTy); 3372651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3373651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // %sum = add <4 x i32> %lhs, %rhs 3374651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 3375651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); 3376651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn"); 3377651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3378651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16> 337987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Constant *ShiftAmt = 338087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2); 3381651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn"); 3382651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3383651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // %res = trunc <4 x i32> %high to <4 x i16> 3384651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], VTy, "vaddhn"); 33852e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu } 3386651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcale_v: 3387651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcaleq_v: 3388651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcalt_v: 3389651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcaltq_v: 3390651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines std::swap(Ops[0], Ops[1]); 3391651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcage_v: 3392651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcageq_v: 3393651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcagt_v: 3394651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcagtq_v: { 3395651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *VecFlt = llvm::VectorType::get( 3396651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy->getScalarSizeInBits() == 32 ? FloatTy : DoubleTy, 3397651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy->getNumElements()); 3398651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[] = { VTy, VecFlt }; 3399651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); 3400651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(F, Ops, NameHint); 3401651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3402651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclz_v: 3403651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclzq_v: 3404651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // We generate target-independent intrinsic, which needs a second argument 3405651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // for whether or not clz of zero is undefined; on ARM it isn't. 3406651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef())); 3407651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 3408651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_f32_v: 3409651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_f32_v: 3410651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3411651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, Quad)); 3412651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") 3413651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); 3414651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_n_f32_v: 3415651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_n_f64_v: 3416651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_n_f32_v: 3417651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_n_f64_v: { 341887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *Tys[2] = { GetFloatNeonType(this, Type), Ty }; 3419651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic; 3420651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(Int, Tys); 3421651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(F, Ops, "vcvt_n"); 34222e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu } 3423651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_n_s32_v: 3424651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_n_u32_v: 3425651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_n_s64_v: 3426651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_n_u64_v: 3427651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_n_s32_v: 3428651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_n_u32_v: 3429651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_n_s64_v: 3430651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_n_u64_v: { 343187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; 3432651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); 3433651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(F, Ops, "vcvt_n"); 34342e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu } 3435651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_s32_v: 3436651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_u32_v: 3437651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_s64_v: 3438651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_u64_v: 3439651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_s32_v: 3440651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_u32_v: 3441651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_s64_v: 3442651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_u64_v: { 344387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type)); 3444651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt") 3445651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : Builder.CreateFPToSI(Ops[0], Ty, "vcvt"); 34462e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu } 3447651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvta_s32_v: 3448651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvta_s64_v: 3449651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvta_u32_v: 3450651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvta_u64_v: 3451651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtaq_s32_v: 3452651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtaq_s64_v: 3453651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtaq_u32_v: 3454651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtaq_u64_v: 3455651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtn_s32_v: 3456651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtn_s64_v: 3457651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtn_u32_v: 3458651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtn_u64_v: 3459651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtnq_s32_v: 3460651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtnq_s64_v: 3461651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtnq_u32_v: 3462651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtnq_u64_v: 3463651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtp_s32_v: 3464651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtp_s64_v: 3465651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtp_u32_v: 3466651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtp_u64_v: 3467651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtpq_s32_v: 3468651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtpq_s64_v: 3469651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtpq_u32_v: 3470651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtpq_u64_v: 3471651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtm_s32_v: 3472651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtm_s64_v: 3473651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtm_u32_v: 3474651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtm_u64_v: 3475651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtmq_s32_v: 3476651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtmq_s64_v: 3477651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtmq_u32_v: 3478651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtmq_u64_v: { 347987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; 3480651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint); 3481651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3482651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vext_v: 3483651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vextq_v: { 3484651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines int CV = cast<ConstantInt>(Ops[2])->getSExtValue(); 34854967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SmallVector<uint32_t, 16> Indices; 3486651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) 34874967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Indices.push_back(i+CV); 34882e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu 3489651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3490651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 34914967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices, "vext"); 34922e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu } 3493651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfma_v: 3494651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfmaq_v: { 3495651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 3496651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3497651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3498651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 34992e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu 3500651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // NEON intrinsic puts accumulator first, unlike the LLVM fma. 3501b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); 3502651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3503651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_v: 350487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NEON::BI__builtin_neon_vld1q_v: { 350587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *Tys[] = {Ty, Int8PtrTy}; 350687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops.push_back(getAlignmentValue32(PtrOp0)); 350787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vld1"); 350887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 3509651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_v: 3510651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2q_v: 3511651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_v: 3512651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3q_v: 3513651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_v: 3514651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4q_v: { 351587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *Tys[] = {Ty, Int8PtrTy}; 351687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); 351787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Align = getAlignmentValue32(PtrOp1); 3518b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint); 3519651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 3520651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 352187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 3522651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3523651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_dup_v: 3524651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_dup_v: { 3525651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *V = UndefValue::get(Ty); 3526651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::PointerType::getUnqual(VTy->getElementType()); 352787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar PtrOp0 = Builder.CreateBitCast(PtrOp0, Ty); 352887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar LoadInst *Ld = Builder.CreateLoad(PtrOp0); 3529c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Constant *CI = ConstantInt::get(SizeTy, 0); 3530651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateInsertElement(V, Ld, CI); 3531651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonSplat(Ops[0], CI); 3532651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3533651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_lane_v: 3534651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2q_lane_v: 3535651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_lane_v: 3536651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3q_lane_v: 3537651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_lane_v: 3538651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4q_lane_v: { 353987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *Tys[] = {Ty, Int8PtrTy}; 354087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); 3541651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines for (unsigned I = 2; I < Ops.size() - 1; ++I) 3542651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[I] = Builder.CreateBitCast(Ops[I], Ty); 354387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops.push_back(getAlignmentValue32(PtrOp1)); 3544651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), NameHint); 3545651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 3546651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 354787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 3548651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3549651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmovl_v: { 3550651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy); 3551651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], DTy); 3552651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Usgn) 3553651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateZExt(Ops[0], Ty, "vmovl"); 3554651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateSExt(Ops[0], Ty, "vmovl"); 3555651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3556651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmovn_v: { 3557651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *QTy = llvm::VectorType::getExtendedElementVectorType(VTy); 3558651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], QTy); 3559651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], Ty, "vmovn"); 3560651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3561651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmull_v: 3562651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: the integer vmull operations could be emitted in terms of pure 3563651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // LLVM IR (2 exts followed by a mul). Unfortunately LLVM has a habit of 3564651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // hoisting the exts outside loops. Until global ISel comes along that can 3565651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // see through such movement this leads to bad CodeGen. So we need an 3566651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // intrinsic for now. 3567651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls; 3568651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp : Int; 3569651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); 3570651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpadal_v: 3571651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpadalq_v: { 3572651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The source operand type has twice as many elements of half the size. 3573651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits(); 3574651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *EltTy = 3575651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), EltBits / 2); 3576651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *NarrowTy = 3577651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(EltTy, VTy->getNumElements() * 2); 3578651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, NarrowTy }; 3579651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint); 3580651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3581651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpaddl_v: 3582651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpaddlq_v: { 3583651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The source operand type has twice as many elements of half the size. 3584651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits(); 3585651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *EltTy = llvm::IntegerType::get(getLLVMContext(), EltBits / 2); 3586651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *NarrowTy = 3587651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(EltTy, VTy->getNumElements() * 2); 3588651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, NarrowTy }; 3589651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl"); 3590651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3591651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlal_v: 3592651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlsl_v: { 3593651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end()); 359487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[1] = 359587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), MulOps, "vqdmlal"); 359687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops.resize(2); 359787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), Ops, NameHint); 3598651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3599651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshl_n_v: 3600651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshlq_n_v: 3601651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n", 3602651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1, false); 3603176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case NEON::BI__builtin_neon_vqshlu_n_v: 3604176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case NEON::BI__builtin_neon_vqshluq_n_v: 3605176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshlu_n", 3606176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 1, false); 3607651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrecpe_v: 3608651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrecpeq_v: 3609651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrsqrte_v: 3610651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrsqrteq_v: 3611651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic; 3612651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint); 3613651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3614176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case NEON::BI__builtin_neon_vrshr_n_v: 3615176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case NEON::BI__builtin_neon_vrshrq_n_v: 3616176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n", 3617176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 1, true); 3618651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshl_n_v: 3619651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshlq_n_v: 3620651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false); 3621651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1], 3622651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vshl_n"); 3623651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshll_n_v: { 3624651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *SrcTy = llvm::VectorType::getTruncatedElementVectorType(VTy); 3625651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 3626651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Usgn) 3627651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateZExt(Ops[0], VTy); 3628651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines else 3629651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateSExt(Ops[0], VTy); 3630651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = EmitNeonShiftVector(Ops[1], VTy, false); 3631651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateShl(Ops[0], Ops[1], "vshll_n"); 3632651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3633651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshrn_n_v: { 3634651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *SrcTy = llvm::VectorType::getExtendedElementVectorType(VTy); 3635651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 3636651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy, false); 3637651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Usgn) 3638651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]); 3639651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines else 3640651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]); 3641651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n"); 3642651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3643651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshr_n_v: 3644651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshrq_n_v: 3645651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn, "vshr_n"); 3646651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_v: 3647651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_v: 3648651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2_v: 3649651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2q_v: 3650651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3_v: 3651651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3q_v: 3652651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4_v: 3653651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4q_v: 3654651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2_lane_v: 3655651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2q_lane_v: 3656651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3_lane_v: 3657651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3q_lane_v: 3658651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4_lane_v: 365987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NEON::BI__builtin_neon_vst4q_lane_v: { 366087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *Tys[] = {Int8PtrTy, Ty}; 366187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops.push_back(getAlignmentValue32(PtrOp0)); 366287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, ""); 366387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 3664651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsubhn_v: { 3665651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType *SrcTy = 3666651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::getExtendedElementVectorType(VTy); 3667651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3668651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // %sum = add <4 x i32> %lhs, %rhs 3669651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 3670651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); 3671651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn"); 3672651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3673651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16> 367487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Constant *ShiftAmt = 367587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2); 3676651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn"); 3677651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3678651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // %res = trunc <4 x i32> %high to <4 x i16> 3679651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateTrunc(Ops[0], VTy, "vsubhn"); 3680651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3681651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtrn_v: 3682651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtrnq_v: { 3683651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 3684651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3685651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 36866bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *SV = nullptr; 3687651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 3688651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines for (unsigned vi = 0; vi != 2; ++vi) { 36894967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SmallVector<uint32_t, 16> Indices; 3690651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 36914967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Indices.push_back(i+vi); 36924967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Indices.push_back(i+e+vi); 3693651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 369458878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); 36954967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn"); 369687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar SV = Builder.CreateDefaultAlignedStore(SV, Addr); 3697651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 36986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return SV; 36996bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } 37006bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case NEON::BI__builtin_neon_vtst_v: 37016bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case NEON::BI__builtin_neon_vtstq_v: { 3702651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3703651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 37046bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); 37056bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], 37066bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ConstantAggregateZero::get(Ty)); 37076bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return Builder.CreateSExt(Ops[0], Ty, "vtst"); 37086bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } 37096bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case NEON::BI__builtin_neon_vuzp_v: 37106bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case NEON::BI__builtin_neon_vuzpq_v: { 37116bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 37126bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 37136bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 37146bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *SV = nullptr; 3715651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 37166bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines for (unsigned vi = 0; vi != 2; ++vi) { 37174967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SmallVector<uint32_t, 16> Indices; 37186bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) 37194967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Indices.push_back(2*i+vi); 3720651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 372158878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); 37224967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp"); 372387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar SV = Builder.CreateDefaultAlignedStore(SV, Addr); 37246bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } 37256bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return SV; 3726651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 37276bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case NEON::BI__builtin_neon_vzip_v: 37286bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case NEON::BI__builtin_neon_vzipq_v: { 37296bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 3730651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3731651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 37326bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *SV = nullptr; 3733651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 37346bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines for (unsigned vi = 0; vi != 2; ++vi) { 37354967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SmallVector<uint32_t, 16> Indices; 37366bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 37374967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Indices.push_back((i + vi*e) >> 1); 37384967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Indices.push_back(((i + vi*e) >> 1)+e); 37396bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } 374058878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); 37414967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip"); 374287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar SV = Builder.CreateDefaultAlignedStore(SV, Addr); 37436bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } 37446bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return SV; 3745651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 3746651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 37476bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 37486bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines assert(Int && "Expected valid intrinsic number"); 37496bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 37506bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // Determine the type(s) of this overloaded AArch64 intrinsic. 37516bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = LookupNeonLLVMIntrinsic(Int, Modifier, Ty, E); 37526bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 37536bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *Result = EmitNeonCall(F, Ops, NameHint); 37546bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines llvm::Type *ResultType = ConvertType(E->getType()); 37556bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // AArch64 intrinsic one-element vector type cast to 37566bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // scalar type expected by the builtin 37576bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return Builder.CreateBitCast(Result, ResultType, NameHint); 37586bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines} 37596bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 37606bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen HinesValue *CodeGenFunction::EmitAArch64CompareBuiltinExpr( 37616bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *Op, llvm::Type *Ty, const CmpInst::Predicate Fp, 37626bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines const CmpInst::Predicate Ip, const Twine &Name) { 37636bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines llvm::Type *OTy = Op->getType(); 37646bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 37656bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // FIXME: this is utterly horrific. We should not be looking at previous 37666bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // codegen context to find out what needs doing. Unfortunately TableGen 37676bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // currently gives us exactly the same calls for vceqz_f32 and vceqz_s32 37686bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // (etc). 37696bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (BitCastInst *BI = dyn_cast<BitCastInst>(Op)) 37706bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines OTy = BI->getOperand(0)->getType(); 37716bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 37726bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Op = Builder.CreateBitCast(Op, OTy); 37736bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (OTy->getScalarType()->isFloatingPointTy()) { 37746bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Op = Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy)); 37756bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } else { 37766bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Op = Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy)); 3777651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 37786bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return Builder.CreateSExt(Op, Ty, Name); 37796bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines} 37806bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 37816bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hinesstatic Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops, 37826bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *ExtOp, Value *IndexOp, 37836bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines llvm::Type *ResTy, unsigned IntID, 37846bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines const char *Name) { 37856bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SmallVector<Value *, 2> TblOps; 37866bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (ExtOp) 37876bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines TblOps.push_back(ExtOp); 37886bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 37894967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Build a vector containing sequential number like (0, 1, 2, ..., 15) 37904967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SmallVector<uint32_t, 16> Indices; 37916bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines llvm::VectorType *TblTy = cast<llvm::VectorType>(Ops[0]->getType()); 37926bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines for (unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) { 37934967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Indices.push_back(2*i); 37944967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Indices.push_back(2*i+1); 3795651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 37966bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 37976bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines int PairPos = 0, End = Ops.size() - 1; 37986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines while (PairPos < End) { 37996bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], 38004967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Ops[PairPos+1], Indices, 38014967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Name)); 38026bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines PairPos += 2; 3803651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 38046bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 38056bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // If there's an odd number of 64-bit lookup table, fill the high 64-bit 38066bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // of the 128-bit lookup table with zero. 38076bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (PairPos == End) { 38086bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *ZeroTbl = ConstantAggregateZero::get(TblTy); 38096bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], 38104967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar ZeroTbl, Indices, Name)); 3811651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 38126bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 38136bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *TblF; 38146bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines TblOps.push_back(IndexOp); 38156bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines TblF = CGF.CGM.getIntrinsic(IntID, ResTy); 38164967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 38176bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return CGF.EmitNeonCall(TblF, TblOps, Name); 3818651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 3819651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 38200e2c34f92f00628d48968dfea096d36381f494cbStephen HinesValue *CodeGenFunction::GetValueForARMHint(unsigned BuiltinID) { 382187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar unsigned Value; 38226bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines switch (BuiltinID) { 38230e2c34f92f00628d48968dfea096d36381f494cbStephen Hines default: 38240e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return nullptr; 3825176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case ARM::BI__builtin_arm_nop: 382687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value = 0; 382787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar break; 3828c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case ARM::BI__builtin_arm_yield: 38296bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case ARM::BI__yield: 383087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value = 1; 383187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar break; 3832c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case ARM::BI__builtin_arm_wfe: 38336bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case ARM::BI__wfe: 383487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value = 2; 383587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar break; 3836c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case ARM::BI__builtin_arm_wfi: 38376bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case ARM::BI__wfi: 383887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value = 3; 383987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar break; 3840c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case ARM::BI__builtin_arm_sev: 38416bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case ARM::BI__sev: 384287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value = 4; 384387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar break; 3844c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case ARM::BI__builtin_arm_sevl: 38456bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case ARM::BI__sevl: 384687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value = 5; 384787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar break; 38486bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } 384987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 385087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint), 385187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::ConstantInt::get(Int32Ty, Value)); 38520e2c34f92f00628d48968dfea096d36381f494cbStephen Hines} 38536bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 3854b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar// Generates the IR for the read/write special register builtin, 3855b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar// ValueType is the type of the value that is to be written or read, 3856b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar// RegisterType is the type of the register being written to or read from. 3857b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainarstatic Value *EmitSpecialRegisterBuiltin(CodeGenFunction &CGF, 3858b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar const CallExpr *E, 3859b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Type *RegisterType, 38604967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Type *ValueType, 38614967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar bool IsRead, 38624967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar StringRef SysReg = "") { 3863b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // write and register intrinsics only support 32 and 64 bit operations. 3864b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64)) 3865b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar && "Unsupported size for register."); 3866b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 3867b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar CodeGen::CGBuilderTy &Builder = CGF.Builder; 3868b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar CodeGen::CodeGenModule &CGM = CGF.CGM; 3869b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar LLVMContext &Context = CGM.getLLVMContext(); 3870b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 38714967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (SysReg.empty()) { 38724967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar const Expr *SysRegStrExpr = E->getArg(0)->IgnoreParenCasts(); 38734967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SysReg = cast<StringLiteral>(SysRegStrExpr)->getString(); 38744967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 3875b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 3876b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) }; 3877b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); 3878b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName); 3879b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 3880b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Type *Types[] = { RegisterType }; 3881b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 3882b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar bool MixedTypes = RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32); 3883b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64)) 3884b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar && "Can't fit 64-bit value in 32-bit register"); 3885b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 3886b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar if (IsRead) { 3887b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::read_register, Types); 3888b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Value *Call = Builder.CreateCall(F, Metadata); 3889b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 3890b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar if (MixedTypes) 3891b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // Read into 64 bit register and then truncate result to 32 bit. 3892b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateTrunc(Call, ValueType); 3893b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 3894b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar if (ValueType->isPointerTy()) 3895b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // Have i32/i64 result (Call) but want to return a VoidPtrTy (i8*). 3896b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateIntToPtr(Call, ValueType); 3897b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 3898b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Call; 3899b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 3900b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 3901b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::write_register, Types); 3902b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Value *ArgValue = CGF.EmitScalarExpr(E->getArg(1)); 3903b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar if (MixedTypes) { 3904b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // Extend 32 bit write value to 64 bit to pass to write. 3905b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar ArgValue = Builder.CreateZExt(ArgValue, RegisterType); 3906b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, { Metadata, ArgValue }); 3907b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 3908b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 3909b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar if (ValueType->isPointerTy()) { 3910b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // Have VoidPtrTy ArgValue but want to return an i32/i64. 3911b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar ArgValue = Builder.CreatePtrToInt(ArgValue, RegisterType); 3912b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, { Metadata, ArgValue }); 3913b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 3914b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 3915b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, { Metadata, ArgValue }); 3916b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar} 3917b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 391887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar/// Return true if BuiltinID is an overloaded Neon intrinsic with an extra 391987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar/// argument that specifies the vector type. 392087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic bool HasExtraNeonArgument(unsigned BuiltinID) { 392187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar switch (BuiltinID) { 392287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar default: break; 392387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NEON::BI__builtin_neon_vget_lane_i8: 392487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NEON::BI__builtin_neon_vget_lane_i16: 392587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NEON::BI__builtin_neon_vget_lane_i32: 392687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NEON::BI__builtin_neon_vget_lane_i64: 392787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NEON::BI__builtin_neon_vget_lane_f32: 392887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NEON::BI__builtin_neon_vgetq_lane_i8: 392987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NEON::BI__builtin_neon_vgetq_lane_i16: 393087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NEON::BI__builtin_neon_vgetq_lane_i32: 393187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NEON::BI__builtin_neon_vgetq_lane_i64: 393287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NEON::BI__builtin_neon_vgetq_lane_f32: 393387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NEON::BI__builtin_neon_vset_lane_i8: 393487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NEON::BI__builtin_neon_vset_lane_i16: 393587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NEON::BI__builtin_neon_vset_lane_i32: 393687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NEON::BI__builtin_neon_vset_lane_i64: 393787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NEON::BI__builtin_neon_vset_lane_f32: 393887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NEON::BI__builtin_neon_vsetq_lane_i8: 393987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NEON::BI__builtin_neon_vsetq_lane_i16: 394087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NEON::BI__builtin_neon_vsetq_lane_i32: 394187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NEON::BI__builtin_neon_vsetq_lane_i64: 394287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NEON::BI__builtin_neon_vsetq_lane_f32: 394387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NEON::BI__builtin_neon_vsha1h_u32: 394487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NEON::BI__builtin_neon_vsha1cq_u32: 394587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NEON::BI__builtin_neon_vsha1pq_u32: 394687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NEON::BI__builtin_neon_vsha1mq_u32: 394787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case ARM::BI_MoveToCoprocessor: 394887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case ARM::BI_MoveToCoprocessor2: 394987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return false; 395087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 395187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return true; 395287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 395387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 39540e2c34f92f00628d48968dfea096d36381f494cbStephen HinesValue *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, 39550e2c34f92f00628d48968dfea096d36381f494cbStephen Hines const CallExpr *E) { 39560e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (auto Hint = GetValueForARMHint(BuiltinID)) 39570e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return Hint; 39580e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 39590e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (BuiltinID == ARM::BI__emit) { 39600e2c34f92f00628d48968dfea096d36381f494cbStephen Hines bool IsThumb = getTarget().getTriple().getArch() == llvm::Triple::thumb; 39610e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::FunctionType *FTy = 39620e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::FunctionType::get(VoidTy, /*Variadic=*/false); 39630e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 39640e2c34f92f00628d48968dfea096d36381f494cbStephen Hines APSInt Value; 39650e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (!E->getArg(0)->EvaluateAsInt(Value, CGM.getContext())) 39660e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm_unreachable("Sema will ensure that the parameter is constant"); 39670e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 39680e2c34f92f00628d48968dfea096d36381f494cbStephen Hines uint64_t ZExtValue = Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue(); 39690e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 39700e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::InlineAsm *Emit = 39710e2c34f92f00628d48968dfea096d36381f494cbStephen Hines IsThumb ? InlineAsm::get(FTy, ".inst.n 0x" + utohexstr(ZExtValue), "", 39720e2c34f92f00628d48968dfea096d36381f494cbStephen Hines /*SideEffects=*/true) 39730e2c34f92f00628d48968dfea096d36381f494cbStephen Hines : InlineAsm::get(FTy, ".inst 0x" + utohexstr(ZExtValue), "", 39740e2c34f92f00628d48968dfea096d36381f494cbStephen Hines /*SideEffects=*/true); 39750e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 397687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateCall(Emit); 39776bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } 39786bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 3979176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (BuiltinID == ARM::BI__builtin_arm_dbg) { 3980176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *Option = EmitScalarExpr(E->getArg(0)); 3981176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_dbg), Option); 3982176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 3983176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 3984176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (BuiltinID == ARM::BI__builtin_arm_prefetch) { 3985176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *Address = EmitScalarExpr(E->getArg(0)); 3986176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *RW = EmitScalarExpr(E->getArg(1)); 3987176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *IsData = EmitScalarExpr(E->getArg(2)); 3988176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 3989176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // Locality is not supported on ARM target 3990176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *Locality = llvm::ConstantInt::get(Int32Ty, 3); 3991176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 3992176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *F = CGM.getIntrinsic(Intrinsic::prefetch); 3993b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, {Address, RW, Locality, IsData}); 3994176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 3995176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 3996c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines if (BuiltinID == ARM::BI__builtin_arm_rbit) { 3997c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_rbit), 3998c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines EmitScalarExpr(E->getArg(0)), 3999c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines "rbit"); 4000c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines } 4001c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 4002651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (BuiltinID == ARM::BI__clear_cache) { 4003651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments"); 4004651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const FunctionDecl *FD = E->getDirectCallee(); 400587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Ops[2]; 4006651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines for (unsigned i = 0; i < 2; i++) 400787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[i] = EmitScalarExpr(E->getArg(i)); 4008651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType()); 4009651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty); 4010651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines StringRef Name = FD->getName(); 4011651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); 4012651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4013651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 40144967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (BuiltinID == ARM::BI__builtin_arm_mcrr || 40154967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar BuiltinID == ARM::BI__builtin_arm_mcrr2) { 40164967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Function *F; 40174967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 40184967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar switch (BuiltinID) { 40194967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar default: llvm_unreachable("unexpected builtin"); 40204967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case ARM::BI__builtin_arm_mcrr: 40214967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar F = CGM.getIntrinsic(Intrinsic::arm_mcrr); 40224967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar break; 40234967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case ARM::BI__builtin_arm_mcrr2: 40244967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar F = CGM.getIntrinsic(Intrinsic::arm_mcrr2); 40254967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar break; 40264967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 40274967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 40284967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // MCRR{2} instruction has 5 operands but 40294967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // the intrinsic has 4 because Rt and Rt2 40304967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // are represented as a single unsigned 64 40314967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // bit integer in the intrinsic definition 40324967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // but internally it's represented as 2 32 40334967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // bit integers. 40344967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 40354967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Coproc = EmitScalarExpr(E->getArg(0)); 40364967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Opc1 = EmitScalarExpr(E->getArg(1)); 40374967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *RtAndRt2 = EmitScalarExpr(E->getArg(2)); 40384967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *CRm = EmitScalarExpr(E->getArg(3)); 40394967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 40404967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *C1 = llvm::ConstantInt::get(Int64Ty, 32); 40414967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Rt = Builder.CreateTruncOrBitCast(RtAndRt2, Int32Ty); 40424967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Rt2 = Builder.CreateLShr(RtAndRt2, C1); 40434967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Rt2 = Builder.CreateTruncOrBitCast(Rt2, Int32Ty); 40444967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 40454967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return Builder.CreateCall(F, {Coproc, Opc1, Rt, Rt2, CRm}); 40464967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 40474967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 40484967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (BuiltinID == ARM::BI__builtin_arm_mrrc || 40494967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar BuiltinID == ARM::BI__builtin_arm_mrrc2) { 40504967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Function *F; 40514967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 40524967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar switch (BuiltinID) { 40534967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar default: llvm_unreachable("unexpected builtin"); 40544967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case ARM::BI__builtin_arm_mrrc: 40554967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar F = CGM.getIntrinsic(Intrinsic::arm_mrrc); 40564967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar break; 40574967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case ARM::BI__builtin_arm_mrrc2: 40584967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar F = CGM.getIntrinsic(Intrinsic::arm_mrrc2); 40594967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar break; 40604967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 40614967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 40624967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Coproc = EmitScalarExpr(E->getArg(0)); 40634967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Opc1 = EmitScalarExpr(E->getArg(1)); 40644967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *CRm = EmitScalarExpr(E->getArg(2)); 40654967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *RtAndRt2 = Builder.CreateCall(F, {Coproc, Opc1, CRm}); 40664967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 40674967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Returns an unsigned 64 bit integer, represented 40684967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // as two 32 bit integers. 40694967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 40704967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Rt = Builder.CreateExtractValue(RtAndRt2, 1); 40714967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Rt1 = Builder.CreateExtractValue(RtAndRt2, 0); 40724967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Rt = Builder.CreateZExt(Rt, Int64Ty); 40734967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Rt1 = Builder.CreateZExt(Rt1, Int64Ty); 40744967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 40754967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *ShiftCast = llvm::ConstantInt::get(Int64Ty, 32); 40764967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar RtAndRt2 = Builder.CreateShl(Rt, ShiftCast, "shl", true); 40774967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar RtAndRt2 = Builder.CreateOr(RtAndRt2, Rt1); 40784967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 40794967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return Builder.CreateBitCast(RtAndRt2, ConvertType(E->getType())); 40804967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 40814967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 4082651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (BuiltinID == ARM::BI__builtin_arm_ldrexd || 4083c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines ((BuiltinID == ARM::BI__builtin_arm_ldrex || 4084c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines BuiltinID == ARM::BI__builtin_arm_ldaex) && 4085c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines getContext().getTypeSize(E->getType()) == 64) || 4086c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines BuiltinID == ARM::BI__ldrexd) { 4087c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Function *F; 4088c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 4089c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines switch (BuiltinID) { 4090c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines default: llvm_unreachable("unexpected builtin"); 4091c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case ARM::BI__builtin_arm_ldaex: 4092c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines F = CGM.getIntrinsic(Intrinsic::arm_ldaexd); 4093c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines break; 4094c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case ARM::BI__builtin_arm_ldrexd: 4095c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case ARM::BI__builtin_arm_ldrex: 4096c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case ARM::BI__ldrexd: 4097c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines F = CGM.getIntrinsic(Intrinsic::arm_ldrexd); 4098c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines break; 4099c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines } 4100651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4101651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *LdPtr = EmitScalarExpr(E->getArg(0)); 4102651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy), 4103651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "ldrexd"); 4104651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4105651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Val0 = Builder.CreateExtractValue(Val, 1); 4106651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Val1 = Builder.CreateExtractValue(Val, 0); 4107651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val0 = Builder.CreateZExt(Val0, Int64Ty); 4108651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val1 = Builder.CreateZExt(Val1, Int64Ty); 4109651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4110651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *ShiftCst = llvm::ConstantInt::get(Int64Ty, 32); 4111651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */); 4112651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val = Builder.CreateOr(Val, Val1); 4113651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateBitCast(Val, ConvertType(E->getType())); 4114651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4115651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4116c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines if (BuiltinID == ARM::BI__builtin_arm_ldrex || 4117c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines BuiltinID == ARM::BI__builtin_arm_ldaex) { 4118651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *LoadAddr = EmitScalarExpr(E->getArg(0)); 4119651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4120651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines QualType Ty = E->getType(); 4121651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *RealResTy = ConvertType(Ty); 4122651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *IntResTy = llvm::IntegerType::get(getLLVMContext(), 4123651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines getContext().getTypeSize(Ty)); 4124651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines LoadAddr = Builder.CreateBitCast(LoadAddr, IntResTy->getPointerTo()); 4125651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4126c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_ldaex 4127c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines ? Intrinsic::arm_ldaex 4128c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines : Intrinsic::arm_ldrex, 4129c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines LoadAddr->getType()); 4130651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Val = Builder.CreateCall(F, LoadAddr, "ldrex"); 4131651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4132651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (RealResTy->isPointerTy()) 4133651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateIntToPtr(Val, RealResTy); 4134651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines else { 4135651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val = Builder.CreateTruncOrBitCast(Val, IntResTy); 4136651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateBitCast(Val, RealResTy); 4137651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4138651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4139651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4140651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (BuiltinID == ARM::BI__builtin_arm_strexd || 4141c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines ((BuiltinID == ARM::BI__builtin_arm_stlex || 4142c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines BuiltinID == ARM::BI__builtin_arm_strex) && 4143651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines getContext().getTypeSize(E->getArg(0)->getType()) == 64)) { 4144c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex 4145c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines ? Intrinsic::arm_stlexd 4146c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines : Intrinsic::arm_strexd); 41470e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, nullptr); 4148651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 414987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Tmp = CreateMemTemp(E->getArg(0)->getType()); 4150651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Val = EmitScalarExpr(E->getArg(0)); 4151651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Builder.CreateStore(Val, Tmp); 4152651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 415387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy)); 4154651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val = Builder.CreateLoad(LdPtr); 4155651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4156651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Arg0 = Builder.CreateExtractValue(Val, 0); 4157651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Arg1 = Builder.CreateExtractValue(Val, 1); 4158651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), Int8PtrTy); 4159b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "strexd"); 41602e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu } 4161651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4162c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines if (BuiltinID == ARM::BI__builtin_arm_strex || 4163c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines BuiltinID == ARM::BI__builtin_arm_stlex) { 4164651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *StoreVal = EmitScalarExpr(E->getArg(0)); 4165651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *StoreAddr = EmitScalarExpr(E->getArg(1)); 4166651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4167651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines QualType Ty = E->getArg(0)->getType(); 4168651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(), 4169651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines getContext().getTypeSize(Ty)); 4170651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo()); 4171651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4172651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (StoreVal->getType()->isPointerTy()) 4173651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines StoreVal = Builder.CreatePtrToInt(StoreVal, Int32Ty); 4174651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines else { 4175651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines StoreVal = Builder.CreateBitCast(StoreVal, StoreTy); 4176651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int32Ty); 4177651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4178651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4179c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex 4180c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines ? Intrinsic::arm_stlex 4181c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines : Intrinsic::arm_strex, 4182c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines StoreAddr->getType()); 4183b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, {StoreVal, StoreAddr}, "strex"); 41842e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu } 4185651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4186651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (BuiltinID == ARM::BI__builtin_arm_clrex) { 4187651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(Intrinsic::arm_clrex); 418887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateCall(F); 41892e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu } 41902e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu 4191651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // CRC32 4192651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic; 4193651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 4194651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case ARM::BI__builtin_arm_crc32b: 4195651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CRCIntrinsicID = Intrinsic::arm_crc32b; break; 4196651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case ARM::BI__builtin_arm_crc32cb: 4197651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CRCIntrinsicID = Intrinsic::arm_crc32cb; break; 4198651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case ARM::BI__builtin_arm_crc32h: 4199651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CRCIntrinsicID = Intrinsic::arm_crc32h; break; 4200651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case ARM::BI__builtin_arm_crc32ch: 4201651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CRCIntrinsicID = Intrinsic::arm_crc32ch; break; 4202651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case ARM::BI__builtin_arm_crc32w: 4203651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case ARM::BI__builtin_arm_crc32d: 4204651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CRCIntrinsicID = Intrinsic::arm_crc32w; break; 4205651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case ARM::BI__builtin_arm_crc32cw: 4206651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case ARM::BI__builtin_arm_crc32cd: 4207651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CRCIntrinsicID = Intrinsic::arm_crc32cw; break; 4208651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 42092e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu 4210651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (CRCIntrinsicID != Intrinsic::not_intrinsic) { 4211651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Arg0 = EmitScalarExpr(E->getArg(0)); 4212651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Arg1 = EmitScalarExpr(E->getArg(1)); 4213aee8e168112a3cbb6282ab4c6b5ae63933053ebfJiangning Liu 4214651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // crc32{c,}d intrinsics are implemnted as two calls to crc32{c,}w 4215651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // intrinsics, hence we need different codegen for these cases. 4216651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (BuiltinID == ARM::BI__builtin_arm_crc32d || 4217651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines BuiltinID == ARM::BI__builtin_arm_crc32cd) { 4218651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *C1 = llvm::ConstantInt::get(Int64Ty, 32); 4219651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Arg1a = Builder.CreateTruncOrBitCast(Arg1, Int32Ty); 4220651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Arg1b = Builder.CreateLShr(Arg1, C1); 4221651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Arg1b = Builder.CreateTruncOrBitCast(Arg1b, Int32Ty); 42222e22f29b92768ea65ac5c26d354226ecc7509311Jiangning Liu 4223651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(CRCIntrinsicID); 4224b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *Res = Builder.CreateCall(F, {Arg0, Arg1a}); 4225b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, {Res, Arg1b}); 4226651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } else { 4227651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Arg1 = Builder.CreateZExtOrBitCast(Arg1, Int32Ty); 4228ff920eec4d449bee560d8d99636ad0eb50cd9d8dTim Northover 4229651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(CRCIntrinsicID); 4230b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, {Arg0, Arg1}); 4231651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4232ff920eec4d449bee560d8d99636ad0eb50cd9d8dTim Northover } 4233ff920eec4d449bee560d8d99636ad0eb50cd9d8dTim Northover 4234b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar if (BuiltinID == ARM::BI__builtin_arm_rsr || 4235b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar BuiltinID == ARM::BI__builtin_arm_rsr64 || 4236b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar BuiltinID == ARM::BI__builtin_arm_rsrp || 4237b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar BuiltinID == ARM::BI__builtin_arm_wsr || 4238b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar BuiltinID == ARM::BI__builtin_arm_wsr64 || 4239b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar BuiltinID == ARM::BI__builtin_arm_wsrp) { 4240b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 4241b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar bool IsRead = BuiltinID == ARM::BI__builtin_arm_rsr || 4242b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar BuiltinID == ARM::BI__builtin_arm_rsr64 || 4243b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar BuiltinID == ARM::BI__builtin_arm_rsrp; 4244b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 4245b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar bool IsPointerBuiltin = BuiltinID == ARM::BI__builtin_arm_rsrp || 4246b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar BuiltinID == ARM::BI__builtin_arm_wsrp; 4247b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 4248b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar bool Is64Bit = BuiltinID == ARM::BI__builtin_arm_rsr64 || 4249b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar BuiltinID == ARM::BI__builtin_arm_wsr64; 4250b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 4251b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Type *ValueType; 4252b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Type *RegisterType; 4253b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar if (IsPointerBuiltin) { 4254b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar ValueType = VoidPtrTy; 4255b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar RegisterType = Int32Ty; 4256b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } else if (Is64Bit) { 4257b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar ValueType = RegisterType = Int64Ty; 4258b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } else { 4259b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar ValueType = RegisterType = Int32Ty; 4260b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 4261b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 4262b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return EmitSpecialRegisterBuiltin(*this, E, RegisterType, ValueType, IsRead); 4263b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 4264b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 4265b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // Find out if any arguments are required to be integer constant 4266b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // expressions. 4267b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar unsigned ICEArguments = 0; 4268b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar ASTContext::GetBuiltinTypeError Error; 4269b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); 4270b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar assert(Error == ASTContext::GE_None && "Should not codegen an error"); 4271b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 427287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar auto getAlignmentValue32 = [&](Address addr) -> Value* { 427387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.getInt32(addr.getAlignment().getQuantity()); 427487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar }; 427587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 427687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address PtrOp0 = Address::invalid(); 427787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address PtrOp1 = Address::invalid(); 4278651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value*, 4> Ops; 427987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar bool HasExtraArg = HasExtraNeonArgument(BuiltinID); 428087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar unsigned NumArgs = E->getNumArgs() - (HasExtraArg ? 1 : 0); 428187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar for (unsigned i = 0, e = NumArgs; i != e; i++) { 4282dd12780e86575795fa912529a911b01e2abc4677Hao Liu if (i == 0) { 4283dd12780e86575795fa912529a911b01e2abc4677Hao Liu switch (BuiltinID) { 4284651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_v: 4285651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_v: 4286651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_lane_v: 4287651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_lane_v: 4288651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_dup_v: 4289651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_dup_v: 4290651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_v: 4291651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_v: 4292651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_lane_v: 4293651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_lane_v: 4294651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2_v: 4295651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2q_v: 4296651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2_lane_v: 4297651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2q_lane_v: 4298651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3_v: 4299651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3q_v: 4300651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3_lane_v: 4301651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3q_lane_v: 4302651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4_v: 4303651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4q_v: 4304651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4_lane_v: 4305651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4q_lane_v: 4306dd12780e86575795fa912529a911b01e2abc4677Hao Liu // Get the alignment for the argument in addition to the value; 4307dd12780e86575795fa912529a911b01e2abc4677Hao Liu // we'll use it later. 430887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar PtrOp0 = EmitPointerWithAlignment(E->getArg(0)); 430987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops.push_back(PtrOp0.getPointer()); 4310dd12780e86575795fa912529a911b01e2abc4677Hao Liu continue; 4311dd12780e86575795fa912529a911b01e2abc4677Hao Liu } 4312dd12780e86575795fa912529a911b01e2abc4677Hao Liu } 4313dd12780e86575795fa912529a911b01e2abc4677Hao Liu if (i == 1) { 4314dd12780e86575795fa912529a911b01e2abc4677Hao Liu switch (BuiltinID) { 4315651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_v: 4316651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2q_v: 4317651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_v: 4318651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3q_v: 4319651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_v: 4320651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4q_v: 4321651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_lane_v: 4322651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2q_lane_v: 4323651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_lane_v: 4324651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3q_lane_v: 4325651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_lane_v: 4326651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4q_lane_v: 4327651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_dup_v: 4328651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_dup_v: 4329651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_dup_v: 4330dd12780e86575795fa912529a911b01e2abc4677Hao Liu // Get the alignment for the argument in addition to the value; 4331dd12780e86575795fa912529a911b01e2abc4677Hao Liu // we'll use it later. 433287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar PtrOp1 = EmitPointerWithAlignment(E->getArg(1)); 433387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops.push_back(PtrOp1.getPointer()); 4334dd12780e86575795fa912529a911b01e2abc4677Hao Liu continue; 4335dd12780e86575795fa912529a911b01e2abc4677Hao Liu } 4336dd12780e86575795fa912529a911b01e2abc4677Hao Liu } 4337b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 4338b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar if ((ICEArguments & (1 << i)) == 0) { 4339b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Ops.push_back(EmitScalarExpr(E->getArg(i))); 4340b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } else { 4341b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // If this is required to be a constant, constant fold it so that we know 4342b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // that the generated intrinsic gets a ConstantInt. 4343b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::APSInt Result; 4344b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext()); 4345b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst; 4346b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); 4347b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 4348b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover } 4349b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover 4350651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 4351651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines default: break; 435287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 4353651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_i8: 4354651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_i16: 4355651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_i32: 4356651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_i64: 4357651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_f32: 4358651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_i8: 4359651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_i16: 4360651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_i32: 4361651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_i64: 4362651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_f32: 436387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane"); 436487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 4365651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_i8: 4366651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_i16: 4367651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_i32: 4368651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_i64: 4369651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_f32: 4370651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_i8: 4371651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_i16: 4372651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_i32: 4373651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_i64: 4374651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_f32: 4375651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 4376651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4377651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsha1h_u32: 4378651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops, 4379651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vsha1h"); 4380651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsha1cq_u32: 4381651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops, 4382651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vsha1h"); 4383651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsha1pq_u32: 4384651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops, 4385651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vsha1h"); 4386651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsha1mq_u32: 4387651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops, 4388651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vsha1h"); 438987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 439087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // The ARM _MoveToCoprocessor builtins put the input register value as 439187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // the first argument, but the LLVM intrinsic expects it as the third one. 439287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case ARM::BI_MoveToCoprocessor: 439387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case ARM::BI_MoveToCoprocessor2: { 43944967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI_MoveToCoprocessor ? 439587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Intrinsic::arm_mcr : Intrinsic::arm_mcr2); 439687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0], 439787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[3], Ops[4], Ops[5]}); 439887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 4399651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4400651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4401b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover // Get the last argument, which specifies the vector type. 440287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar assert(HasExtraArg); 4403b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover llvm::APSInt Result; 4404651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const Expr *Arg = E->getArg(E->getNumArgs()-1); 4405b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover if (!Arg->isIntegerConstantExpr(Result, getContext())) 44066bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 4407b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover 4408651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f || 4409651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines BuiltinID == ARM::BI__builtin_arm_vcvtr_d) { 4410651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Determine the overloaded type of this builtin. 4411651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty; 4412651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f) 4413651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = FloatTy; 4414912502b4996b14db31b498cb1eef2b17d7d66d57Hao Liu else 4415651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = DoubleTy; 4416912502b4996b14db31b498cb1eef2b17d7d66d57Hao Liu 4417651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Determine whether this is an unsigned conversion or not. 4418651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool usgn = Result.getZExtValue() == 1; 4419651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr; 4420b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover 4421651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Call the appropriate intrinsic. 4422dd12780e86575795fa912529a911b01e2abc4677Hao Liu Function *F = CGM.getIntrinsic(Int, Ty); 4423651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall(F, Ops, "vcvtr"); 4424dd12780e86575795fa912529a911b01e2abc4677Hao Liu } 4425651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4426651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Determine the type of this overloaded NEON intrinsic. 4427651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags Type(Result.getZExtValue()); 4428651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool usgn = Type.isUnsigned(); 4429651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool rightShift = false; 4430651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4431651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType *VTy = GetNeonType(this, Type); 4432651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty = VTy; 4433651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (!Ty) 44346bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 4435651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4436651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Many NEON builtins have identical semantics and uses in ARM and 4437651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // AArch64. Emit these in a single function. 4438176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto IntrinsicMap = makeArrayRef(ARMSIMDIntrinsicMap); 4439651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap( 4440651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines IntrinsicMap, BuiltinID, NEONSIMDIntrinsicsProvenSorted); 4441651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Builtin) 4442651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitCommonNeonBuiltinExpr( 4443651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic, 444487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1); 4445651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4446651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Int; 4447651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 44486bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines default: return nullptr; 4449651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_lane_v: 4450651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Handle 64-bit integer elements as a special case. Use shuffles of 4451651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // one-element vectors to avoid poor code for i64 in the backend. 4452651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (VTy->getElementType()->isIntegerTy(64)) { 4453651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Extract the other lane. 4454651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 445587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar uint32_t Lane = cast<ConstantInt>(Ops[2])->getZExtValue(); 4456651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *SV = llvm::ConstantVector::get(ConstantInt::get(Int32Ty, 1-Lane)); 4457651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); 4458651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Load the value as a one-element vector. 4459651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::VectorType::get(VTy->getElementType(), 1); 446087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *Tys[] = {Ty, Int8PtrTy}; 446187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Tys); 446287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Align = getAlignmentValue32(PtrOp0); 4463b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *Ld = Builder.CreateCall(F, {Ops[0], Align}); 4464651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Combine them. 446587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar uint32_t Indices[] = {1 - Lane, Lane}; 446687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar SV = llvm::ConstantDataVector::get(getLLVMContext(), Indices); 4467651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateShuffleVector(Ops[1], Ld, SV, "vld1q_lane"); 4468dd12780e86575795fa912529a911b01e2abc4677Hao Liu } 4469651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // fall through 4470651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_lane_v: { 447111a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 447287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar PtrOp0 = Builder.CreateElementBitCast(PtrOp0, VTy->getElementType()); 447387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Ld = Builder.CreateLoad(PtrOp0); 447411a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane"); 447511a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu } 4476651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_dup_v: 4477651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_dup_v: 4478651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_dup_v: { 4479651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Handle 64-bit elements as a special-case. There is no "dup" needed. 4480651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (VTy->getElementType()->getPrimitiveSizeInBits() == 64) { 448111a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu switch (BuiltinID) { 4482651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_dup_v: 448311a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Int = Intrinsic::arm_neon_vld2; 448411a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu break; 4485651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_dup_v: 448611a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Int = Intrinsic::arm_neon_vld3; 448711a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu break; 4488651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_dup_v: 448911a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Int = Intrinsic::arm_neon_vld4; 449011a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu break; 4491651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines default: llvm_unreachable("unknown vld_dup intrinsic?"); 449211a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu } 449387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *Tys[] = {Ty, Int8PtrTy}; 449487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Function *F = CGM.getIntrinsic(Int, Tys); 449587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Value *Align = getAlignmentValue32(PtrOp1); 4496b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, "vld_dup"); 449711a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 449811a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 449987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 450011a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu } 450111a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu switch (BuiltinID) { 4502651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_dup_v: 450311a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Int = Intrinsic::arm_neon_vld2lane; 450411a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu break; 4505651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_dup_v: 450611a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Int = Intrinsic::arm_neon_vld3lane; 450711a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu break; 4508651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_dup_v: 450911a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Int = Intrinsic::arm_neon_vld4lane; 451011a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu break; 4511651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines default: llvm_unreachable("unknown vld_dup intrinsic?"); 451211a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu } 451387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *Tys[] = {Ty, Int8PtrTy}; 451487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Function *F = CGM.getIntrinsic(Int, Tys); 451511a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu llvm::StructType *STy = cast<llvm::StructType>(F->getReturnType()); 451611a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu 4517651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value*, 6> Args; 451811a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Args.push_back(Ops[1]); 451911a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Args.append(STy->getNumElements(), UndefValue::get(Ty)); 452011a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu 452111a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu llvm::Constant *CI = ConstantInt::get(Int32Ty, 0); 452211a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Args.push_back(CI); 452387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Args.push_back(getAlignmentValue32(PtrOp1)); 452411a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu 452511a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ops[1] = Builder.CreateCall(F, Args, "vld_dup"); 452611a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu // splat lane 0 to all elts in each vector of the result. 452711a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { 452811a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Value *Val = Builder.CreateExtractValue(Ops[1], i); 452911a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Value *Elt = Builder.CreateBitCast(Val, Ty); 453011a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Elt = EmitNeonSplat(Elt, CI); 453111a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Elt = Builder.CreateBitCast(Elt, Val->getType()); 453211a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ops[1] = Builder.CreateInsertValue(Ops[1], Elt, i); 453311a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu } 453411a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 453511a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 453687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 453711a94f9ccaf29bb7cd073787d5cb6d130a38bf62Hao Liu } 4538651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqrshrn_n_v: 4539651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = 4540651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns; 4541651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n", 4542651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1, true); 4543651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqrshrun_n_v: 4544651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqrshiftnsu, Ty), 4545651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vqrshrun_n", 1, true); 4546651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshrn_n_v: 4547651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns; 4548651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n", 4549651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 1, true); 4550651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshrun_n_v: 4551651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftnsu, Ty), 4552651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vqshrun_n", 1, true); 4553651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrecpe_v: 4554651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrecpeq_v: 4555651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrecpe, Ty), 4556651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vrecpe"); 4557651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrshrn_n_v: 4558651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrshiftn, Ty), 4559651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vrshrn_n", 1, true); 4560651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrsra_n_v: 4561651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrsraq_n_v: 45620aa1a88e19235574481e46e9e6e9ce66a9e6624fJiangning Liu Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 45630aa1a88e19235574481e46e9e6e9ce66a9e6624fJiangning Liu Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 4564651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true); 4565651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts; 4566b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Ty), {Ops[1], Ops[2]}); 4567651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n"); 4568651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsri_n_v: 4569651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsriq_n_v: 4570651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines rightShift = true; 4571651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsli_n_v: 4572651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsliq_n_v: 4573651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift); 4574651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftins, Ty), 4575651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vsli_n"); 4576651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsra_n_v: 4577651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsraq_n_v: 45780aa1a88e19235574481e46e9e6e9ce66a9e6624fJiangning Liu Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4579651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); 4580651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Ops[0], Ops[1]); 4581651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_lane_v: 4582651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Handle 64-bit integer elements as a special case. Use a shuffle to get 4583651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // a one-element vector and avoid poor code for i64 in the backend. 4584651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (VTy->getElementType()->isIntegerTy(64)) { 4585651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 4586651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2])); 4587651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); 458887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[2] = getAlignmentValue32(PtrOp0); 458987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()}; 4590651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1, 459187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Tys), Ops); 459251cc0172a173599b769968696e20638754d1dcd6Ana Pazos } 4593651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // fall through 4594651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_lane_v: { 4595b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 4596651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); 4597651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 459887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar auto St = Builder.CreateStore(Ops[1], Builder.CreateBitCast(PtrOp0, Ty)); 4599651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return St; 46008137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 4601651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl1_v: 4602651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl1), 4603651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbl1"); 4604651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl2_v: 4605651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl2), 4606651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbl2"); 4607651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl3_v: 4608651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl3), 4609651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbl3"); 4610651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl4_v: 4611651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl4), 4612651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbl4"); 4613651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx1_v: 4614651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx1), 4615651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbx1"); 4616651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx2_v: 4617651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx2), 4618651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbx2"); 4619651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx3_v: 4620651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx3), 4621651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbx3"); 4622651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx4_v: 4623651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx4), 4624651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbx4"); 46258137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 4626651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 4627651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 46286bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hinesstatic Value *EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID, 4629651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const CallExpr *E, 4630651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVectorImpl<Value *> &Ops) { 4631651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned int Int = 0; 46326bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines const char *s = nullptr; 4633651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4634651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 4635651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines default: 46366bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 4637651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl1_v: 4638651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl1_v: 4639651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl1q_v: 4640651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl2_v: 4641651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl2_v: 4642651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl2q_v: 4643651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl3_v: 4644651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl3_v: 4645651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl3q_v: 4646651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl4_v: 4647651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl4_v: 4648651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl4q_v: 4649651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 4650651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx1_v: 4651651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx1_v: 4652651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx1q_v: 4653651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx2_v: 4654651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx2_v: 4655651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx2q_v: 4656651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx3_v: 4657651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx3_v: 4658651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx3q_v: 4659651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx4_v: 4660651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx4_v: 4661651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx4q_v: 4662651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 46638137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 4664651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4665651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines assert(E->getNumArgs() >= 3); 4666651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4667651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Get the last argument, which specifies the vector type. 4668651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::APSInt Result; 4669651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const Expr *Arg = E->getArg(E->getNumArgs() - 1); 4670651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (!Arg->isIntegerConstantExpr(Result, CGF.getContext())) 46716bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 4672651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4673651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Determine the type of this overloaded NEON intrinsic. 4674651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags Type(Result.getZExtValue()); 467587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::VectorType *Ty = GetNeonType(&CGF, Type); 4676651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (!Ty) 46776bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 4678651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4679651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CodeGen::CGBuilderTy &Builder = CGF.Builder; 4680651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4681651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // AArch64 scalar builtins are not overloaded, they do not have an extra 4682651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // argument that specifies the vector type, need to handle each case. 4683651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 4684651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl1_v: { 468587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 1), nullptr, 468687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[1], Ty, Intrinsic::aarch64_neon_tbl1, 468787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar "vtbl1"); 46888137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 4689651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl2_v: { 469087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 2), nullptr, 469187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[2], Ty, Intrinsic::aarch64_neon_tbl1, 469287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar "vtbl1"); 46938137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 4694651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl3_v: { 469587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 3), nullptr, 469687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[3], Ty, Intrinsic::aarch64_neon_tbl2, 469787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar "vtbl2"); 46988137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 4699651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbl4_v: { 470087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 4), nullptr, 470187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[4], Ty, Intrinsic::aarch64_neon_tbl2, 470287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar "vtbl2"); 47038137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 4704651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx1_v: { 470587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *TblRes = 470687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 1), nullptr, Ops[2], 470787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty, Intrinsic::aarch64_neon_tbl1, "vtbl1"); 4708651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 470987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Constant *EightV = ConstantInt::get(Ty, 8); 4710651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV); 4711651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CmpRes = Builder.CreateSExt(CmpRes, Ty); 4712651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4713651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); 4714651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes); 4715651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx"); 47168137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 4717651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx2_v: { 471887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 2), Ops[0], 471987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[3], Ty, Intrinsic::aarch64_neon_tbx1, 472087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar "vtbx1"); 47218137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 4722651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx3_v: { 472387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *TblRes = 472487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 3), nullptr, Ops[4], 472587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty, Intrinsic::aarch64_neon_tbl2, "vtbl2"); 472687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 472787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24); 4728651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4], 4729651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TwentyFourV); 4730651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CmpRes = Builder.CreateSExt(CmpRes, Ty); 4731651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4732651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); 4733651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes); 4734651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx"); 47358137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 4736651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtbx4_v: { 473787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 4), Ops[0], 473887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[5], Ty, Intrinsic::aarch64_neon_tbx2, 473987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar "vtbx2"); 4740651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4741651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl1_v: 4742651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl1q_v: 47436bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_tbl1; s = "vtbl1"; break; 4744651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl2_v: 4745651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl2q_v: { 47466bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_tbl2; s = "vtbl2"; break; 4747651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl3_v: 4748651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl3q_v: 47496bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_tbl3; s = "vtbl3"; break; 4750651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl4_v: 4751651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl4q_v: 47526bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_tbl4; s = "vtbl4"; break; 4753651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx1_v: 4754651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx1q_v: 47556bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_tbx1; s = "vtbx1"; break; 4756651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx2_v: 4757651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx2q_v: 47586bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_tbx2; s = "vtbx2"; break; 4759651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx3_v: 4760651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx3q_v: 47616bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_tbx3; s = "vtbx3"; break; 4762651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx4_v: 4763651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx4q_v: 47646bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_tbx4; s = "vtbx4"; break; 47658137a607eebe799d95fd05226fb91119a5b054b0Kevin Qin } 4766b793f0d3448a15277cd6b6cc4ba558ded39a8084Tim Northover } 4767651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4768651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (!Int) 47696bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 4770651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4771651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGF.CGM.getIntrinsic(Int, Ty); 4772651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return CGF.EmitNeonCall(F, Ops, s); 4773ff920eec4d449bee560d8d99636ad0eb50cd9d8dTim Northover} 4774ff920eec4d449bee560d8d99636ad0eb50cd9d8dTim Northover 4775651f13cea278ec967336033dd032faef0e9fc2ecStephen HinesValue *CodeGenFunction::vectorWrapScalar16(Value *Op) { 4776651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *VTy = llvm::VectorType::get(Int16Ty, 4); 4777651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Op = Builder.CreateBitCast(Op, Int16Ty); 4778651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *V = UndefValue::get(VTy); 4779c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Constant *CI = ConstantInt::get(SizeTy, 0); 4780651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Op = Builder.CreateInsertElement(V, Op, CI); 4781651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Op; 4782651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 4783651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 47846bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen HinesValue *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, 47856bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines const CallExpr *E) { 4786176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines unsigned HintID = static_cast<unsigned>(-1); 4787176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines switch (BuiltinID) { 4788176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines default: break; 4789176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case AArch64::BI__builtin_arm_nop: 4790176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines HintID = 0; 4791176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines break; 4792176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case AArch64::BI__builtin_arm_yield: 4793176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines HintID = 1; 4794176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines break; 4795176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case AArch64::BI__builtin_arm_wfe: 4796176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines HintID = 2; 4797176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines break; 4798176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case AArch64::BI__builtin_arm_wfi: 4799176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines HintID = 3; 4800176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines break; 4801176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case AArch64::BI__builtin_arm_sev: 4802176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines HintID = 4; 4803176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines break; 4804176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case AArch64::BI__builtin_arm_sevl: 4805176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines HintID = 5; 4806176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines break; 4807176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 4808176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 4809176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (HintID != static_cast<unsigned>(-1)) { 4810176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_hint); 4811176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID)); 4812176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 4813176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 4814176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (BuiltinID == AArch64::BI__builtin_arm_prefetch) { 4815176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *Address = EmitScalarExpr(E->getArg(0)); 4816176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *RW = EmitScalarExpr(E->getArg(1)); 4817176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *CacheLevel = EmitScalarExpr(E->getArg(2)); 4818176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *RetentionPolicy = EmitScalarExpr(E->getArg(3)); 4819176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *IsData = EmitScalarExpr(E->getArg(4)); 4820176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 4821176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *Locality = nullptr; 4822176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (cast<llvm::ConstantInt>(RetentionPolicy)->isZero()) { 4823176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // Temporal fetch, needs to convert cache level to locality. 4824176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Locality = llvm::ConstantInt::get(Int32Ty, 4825176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines -cast<llvm::ConstantInt>(CacheLevel)->getValue() + 3); 4826176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } else { 4827176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // Streaming fetch. 4828176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Locality = llvm::ConstantInt::get(Int32Ty, 0); 4829176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 4830176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 4831176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // FIXME: We need AArch64 specific LLVM intrinsic if we want to specify 4832176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // PLDL3STRM or PLDL2STRM. 4833176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Value *F = CGM.getIntrinsic(Intrinsic::prefetch); 4834b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, {Address, RW, Locality, IsData}); 4835176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 4836176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 4837c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines if (BuiltinID == AArch64::BI__builtin_arm_rbit) { 4838c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines assert((getContext().getTypeSize(E->getType()) == 32) && 4839c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines "rbit of unusual size!"); 4840c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Arg = EmitScalarExpr(E->getArg(0)); 4841c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines return Builder.CreateCall( 4842c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines CGM.getIntrinsic(Intrinsic::aarch64_rbit, Arg->getType()), Arg, "rbit"); 4843c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines } 4844c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines if (BuiltinID == AArch64::BI__builtin_arm_rbit64) { 4845c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines assert((getContext().getTypeSize(E->getType()) == 64) && 4846c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines "rbit of unusual size!"); 4847c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Arg = EmitScalarExpr(E->getArg(0)); 4848c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines return Builder.CreateCall( 4849c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines CGM.getIntrinsic(Intrinsic::aarch64_rbit, Arg->getType()), Arg, "rbit"); 4850c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines } 4851c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 48526bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (BuiltinID == AArch64::BI__clear_cache) { 48534537d6e0f9baf2e011a4260e0d7872789b01c3f2Rafael Espindola assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments"); 485479ba509b0106fd0a1ff832baeb1fdb5430527efeRafael Espindola const FunctionDecl *FD = E->getDirectCallee(); 485587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Ops[2]; 48564537d6e0f9baf2e011a4260e0d7872789b01c3f2Rafael Espindola for (unsigned i = 0; i < 2; i++) 485787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[i] = EmitScalarExpr(E->getArg(i)); 48582acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType()); 48592acc6e3feda5e4f7d9009bdcf8b1cd777fecfe2dChris Lattner llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty); 48605f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner StringRef Name = FD->getName(); 4861bd7370a78604e9a20d698bfe328c1e43f12a0613John McCall return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); 48622752c0137d95aa2f4ee1cdff4b564bac842e041bChris Lattner } 4863e140af3e27016f902146023fba7680b43043ec07Rafael Espindola 4864c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines if ((BuiltinID == AArch64::BI__builtin_arm_ldrex || 4865c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines BuiltinID == AArch64::BI__builtin_arm_ldaex) && 4866651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines getContext().getTypeSize(E->getType()) == 128) { 4867c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex 4868c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines ? Intrinsic::aarch64_ldaxp 4869c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines : Intrinsic::aarch64_ldxp); 487026c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes 487126c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes Value *LdPtr = EmitScalarExpr(E->getArg(0)); 487209df2b066221d869f17f4b5762405f111a65f983Tim Northover Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy), 4873651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "ldxp"); 487426c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes 487526c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes Value *Val0 = Builder.CreateExtractValue(Val, 1); 487626c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes Value *Val1 = Builder.CreateExtractValue(Val, 0); 4877651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128); 4878651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val0 = Builder.CreateZExt(Val0, Int128Ty); 4879651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val1 = Builder.CreateZExt(Val1, Int128Ty); 488026c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes 4881651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64); 488226c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */); 488309df2b066221d869f17f4b5762405f111a65f983Tim Northover Val = Builder.CreateOr(Val, Val1); 488409df2b066221d869f17f4b5762405f111a65f983Tim Northover return Builder.CreateBitCast(Val, ConvertType(E->getType())); 4885c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines } else if (BuiltinID == AArch64::BI__builtin_arm_ldrex || 4886c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines BuiltinID == AArch64::BI__builtin_arm_ldaex) { 488709df2b066221d869f17f4b5762405f111a65f983Tim Northover Value *LoadAddr = EmitScalarExpr(E->getArg(0)); 488809df2b066221d869f17f4b5762405f111a65f983Tim Northover 488909df2b066221d869f17f4b5762405f111a65f983Tim Northover QualType Ty = E->getType(); 489009df2b066221d869f17f4b5762405f111a65f983Tim Northover llvm::Type *RealResTy = ConvertType(Ty); 489109df2b066221d869f17f4b5762405f111a65f983Tim Northover llvm::Type *IntResTy = llvm::IntegerType::get(getLLVMContext(), 489209df2b066221d869f17f4b5762405f111a65f983Tim Northover getContext().getTypeSize(Ty)); 489309df2b066221d869f17f4b5762405f111a65f983Tim Northover LoadAddr = Builder.CreateBitCast(LoadAddr, IntResTy->getPointerTo()); 489409df2b066221d869f17f4b5762405f111a65f983Tim Northover 4895c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex 4896c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines ? Intrinsic::aarch64_ldaxr 4897c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines : Intrinsic::aarch64_ldxr, 4898c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines LoadAddr->getType()); 4899651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Val = Builder.CreateCall(F, LoadAddr, "ldxr"); 490009df2b066221d869f17f4b5762405f111a65f983Tim Northover 490109df2b066221d869f17f4b5762405f111a65f983Tim Northover if (RealResTy->isPointerTy()) 490209df2b066221d869f17f4b5762405f111a65f983Tim Northover return Builder.CreateIntToPtr(Val, RealResTy); 4903651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 4904651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Val = Builder.CreateTruncOrBitCast(Val, IntResTy); 4905651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateBitCast(Val, RealResTy); 490609df2b066221d869f17f4b5762405f111a65f983Tim Northover } 490709df2b066221d869f17f4b5762405f111a65f983Tim Northover 4908c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines if ((BuiltinID == AArch64::BI__builtin_arm_strex || 4909c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines BuiltinID == AArch64::BI__builtin_arm_stlex) && 4910651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines getContext().getTypeSize(E->getArg(0)->getType()) == 128) { 4911c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex 4912c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines ? Intrinsic::aarch64_stlxp 4913c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines : Intrinsic::aarch64_stxp); 49140e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Type *STy = llvm::StructType::get(Int64Ty, Int64Ty, nullptr); 491526c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes 491687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Tmp = CreateMemTemp(E->getArg(0)->getType()); 491787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar EmitAnyExprToMem(E->getArg(0), Tmp, Qualifiers(), /*init*/ true); 491826c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes 491987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Tmp = Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(STy)); 492087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Value *Val = Builder.CreateLoad(Tmp); 492126c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes 492226c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes Value *Arg0 = Builder.CreateExtractValue(Val, 0); 492326c1b8df8d1af0d8ef7f6c726fe1a8a9ddc60267Bruno Cardoso Lopes Value *Arg1 = Builder.CreateExtractValue(Val, 1); 4924651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), 4925651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int8PtrTy); 4926b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "stxp"); 4927b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 4928b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 4929b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar if (BuiltinID == AArch64::BI__builtin_arm_strex || 4930b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar BuiltinID == AArch64::BI__builtin_arm_stlex) { 493109df2b066221d869f17f4b5762405f111a65f983Tim Northover Value *StoreVal = EmitScalarExpr(E->getArg(0)); 493209df2b066221d869f17f4b5762405f111a65f983Tim Northover Value *StoreAddr = EmitScalarExpr(E->getArg(1)); 493309df2b066221d869f17f4b5762405f111a65f983Tim Northover 493409df2b066221d869f17f4b5762405f111a65f983Tim Northover QualType Ty = E->getArg(0)->getType(); 493509df2b066221d869f17f4b5762405f111a65f983Tim Northover llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(), 493609df2b066221d869f17f4b5762405f111a65f983Tim Northover getContext().getTypeSize(Ty)); 493709df2b066221d869f17f4b5762405f111a65f983Tim Northover StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo()); 493809df2b066221d869f17f4b5762405f111a65f983Tim Northover 493909df2b066221d869f17f4b5762405f111a65f983Tim Northover if (StoreVal->getType()->isPointerTy()) 4940651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines StoreVal = Builder.CreatePtrToInt(StoreVal, Int64Ty); 494109df2b066221d869f17f4b5762405f111a65f983Tim Northover else { 494209df2b066221d869f17f4b5762405f111a65f983Tim Northover StoreVal = Builder.CreateBitCast(StoreVal, StoreTy); 4943651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int64Ty); 494409df2b066221d869f17f4b5762405f111a65f983Tim Northover } 494509df2b066221d869f17f4b5762405f111a65f983Tim Northover 4946c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex 4947c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines ? Intrinsic::aarch64_stlxr 4948c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines : Intrinsic::aarch64_stxr, 4949c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines StoreAddr->getType()); 4950b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, {StoreVal, StoreAddr}, "stxr"); 495109df2b066221d869f17f4b5762405f111a65f983Tim Northover } 495209df2b066221d869f17f4b5762405f111a65f983Tim Northover 49536bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (BuiltinID == AArch64::BI__builtin_arm_clrex) { 49546bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_clrex); 495587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateCall(F); 495687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 495787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 4958520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly // CRC32 4959520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic; 4960520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly switch (BuiltinID) { 49616bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case AArch64::BI__builtin_arm_crc32b: 49626bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines CRCIntrinsicID = Intrinsic::aarch64_crc32b; break; 49636bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case AArch64::BI__builtin_arm_crc32cb: 49646bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines CRCIntrinsicID = Intrinsic::aarch64_crc32cb; break; 49656bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case AArch64::BI__builtin_arm_crc32h: 49666bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines CRCIntrinsicID = Intrinsic::aarch64_crc32h; break; 49676bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case AArch64::BI__builtin_arm_crc32ch: 49686bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines CRCIntrinsicID = Intrinsic::aarch64_crc32ch; break; 49696bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case AArch64::BI__builtin_arm_crc32w: 49706bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines CRCIntrinsicID = Intrinsic::aarch64_crc32w; break; 49716bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case AArch64::BI__builtin_arm_crc32cw: 49726bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines CRCIntrinsicID = Intrinsic::aarch64_crc32cw; break; 49736bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case AArch64::BI__builtin_arm_crc32d: 49746bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines CRCIntrinsicID = Intrinsic::aarch64_crc32x; break; 49756bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines case AArch64::BI__builtin_arm_crc32cd: 49766bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines CRCIntrinsicID = Intrinsic::aarch64_crc32cx; break; 4977520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly } 4978520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly 4979520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly if (CRCIntrinsicID != Intrinsic::not_intrinsic) { 4980520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly Value *Arg0 = EmitScalarExpr(E->getArg(0)); 4981520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly Value *Arg1 = EmitScalarExpr(E->getArg(1)); 4982651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(CRCIntrinsicID); 4983520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly 4984651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *DataTy = F->getFunctionType()->getParamType(1); 4985651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Arg1 = Builder.CreateZExtOrBitCast(Arg1, DataTy); 4986520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly 4987b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, {Arg0, Arg1}); 4988520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly } 4989520ec1e553ffa955276462e4230c1b128fa433afJoey Gouly 4990b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar if (BuiltinID == AArch64::BI__builtin_arm_rsr || 4991b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar BuiltinID == AArch64::BI__builtin_arm_rsr64 || 4992b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar BuiltinID == AArch64::BI__builtin_arm_rsrp || 4993b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar BuiltinID == AArch64::BI__builtin_arm_wsr || 4994b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar BuiltinID == AArch64::BI__builtin_arm_wsr64 || 4995b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar BuiltinID == AArch64::BI__builtin_arm_wsrp) { 4996b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 4997b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar bool IsRead = BuiltinID == AArch64::BI__builtin_arm_rsr || 4998b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar BuiltinID == AArch64::BI__builtin_arm_rsr64 || 4999b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar BuiltinID == AArch64::BI__builtin_arm_rsrp; 5000b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 5001b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar bool IsPointerBuiltin = BuiltinID == AArch64::BI__builtin_arm_rsrp || 5002b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar BuiltinID == AArch64::BI__builtin_arm_wsrp; 5003b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 5004b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar bool Is64Bit = BuiltinID != AArch64::BI__builtin_arm_rsr && 5005b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar BuiltinID != AArch64::BI__builtin_arm_wsr; 5006b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 5007b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Type *ValueType; 5008b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Type *RegisterType = Int64Ty; 5009b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar if (IsPointerBuiltin) { 5010b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar ValueType = VoidPtrTy; 5011b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } else if (Is64Bit) { 5012b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar ValueType = Int64Ty; 5013b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } else { 5014b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar ValueType = Int32Ty; 5015b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 5016b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 5017b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return EmitSpecialRegisterBuiltin(*this, E, RegisterType, ValueType, IsRead); 5018b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 5019b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 5020b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // Find out if any arguments are required to be integer constant 5021b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // expressions. 5022b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar unsigned ICEArguments = 0; 5023b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar ASTContext::GetBuiltinTypeError Error; 5024b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); 5025b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar assert(Error == ASTContext::GE_None && "Should not codegen an error"); 5026b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 5027651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::SmallVector<Value*, 4> Ops; 5028b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) { 5029b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar if ((ICEArguments & (1 << i)) == 0) { 5030b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Ops.push_back(EmitScalarExpr(E->getArg(i))); 5031b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } else { 5032b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // If this is required to be a constant, constant fold it so that we know 5033b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // that the generated intrinsic gets a ConstantInt. 5034b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::APSInt Result; 5035b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext()); 5036b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar assert(IsConst && "Constant arg isn't actually constant?"); 5037b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar (void)IsConst; 5038b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); 5039b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 5040b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 5041651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 5042176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto SISDMap = makeArrayRef(AArch64SISDIntrinsicMap); 5043651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap( 50446bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SISDMap, BuiltinID, AArch64SISDIntrinsicsProvenSorted); 5045651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 5046651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Builtin) { 5047651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(E->getNumArgs() - 1))); 5048651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Result = EmitCommonNeonSISDBuiltinExpr(*this, *Builtin, Ops, E); 5049651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines assert(Result && "SISD intrinsic should have been handled"); 5050651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Result; 5051ea93e40785ffeadfac66b948c95f9490ec26207aEli Friedman } 5052e140af3e27016f902146023fba7680b43043ec07Rafael Espindola 5053651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::APSInt Result; 5054651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines const Expr *Arg = E->getArg(E->getNumArgs()-1); 5055651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags Type(0); 5056651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Arg->isIntegerConstantExpr(Result, getContext())) 5057651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Determine the type of this overloaded NEON intrinsic. 5058651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Type = NeonTypeFlags(Result.getZExtValue()); 5059651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 5060651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool usgn = Type.isUnsigned(); 5061651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool quad = Type.isQuad(); 5062651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 5063651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Handle non-overloaded intrinsics first. 506483bbba1fc98dd3b2a351013a40808e995e648f40Bob Wilson switch (BuiltinID) { 506583bbba1fc98dd3b2a351013a40808e995e648f40Bob Wilson default: break; 5066651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vldrq_p128: { 5067651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128); 5068651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Ptr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), Int128PTy); 506987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateDefaultAlignedLoad(Ptr); 5070651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5071651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vstrq_p128: { 5072651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128); 5073651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy); 507487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateDefaultAlignedStore(EmitScalarExpr(E->getArg(1)), Ptr); 5075651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5076651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvts_u32_f32: 5077651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtd_u64_f64: 5078651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines usgn = true; 5079651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FALL THROUGH 5080651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvts_s32_f32: 5081651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtd_s64_f64: { 5082651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5083651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64; 5084651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty; 5085651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *FTy = Is64 ? DoubleTy : FloatTy; 5086651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], FTy); 5087651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (usgn) 5088651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFPToUI(Ops[0], InTy); 5089651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFPToSI(Ops[0], InTy); 5090651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5091651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvts_f32_u32: 5092651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtd_f64_u64: 5093651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines usgn = true; 5094651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FALL THROUGH 5095651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvts_f32_s32: 5096651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtd_f64_s64: { 5097651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5098651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64; 5099651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty; 5100651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *FTy = Is64 ? DoubleTy : FloatTy; 5101651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], InTy); 5102651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (usgn) 5103651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateUIToFP(Ops[0], FTy); 5104651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateSIToFP(Ops[0], FTy); 5105651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5106651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpaddd_s64: { 510787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *Ty = llvm::VectorType::get(Int64Ty, 2); 5108651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Vec = EmitScalarExpr(E->getArg(0)); 5109651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The vector is v2f64, so make sure it's bitcast to that. 5110651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Vec = Builder.CreateBitCast(Vec, Ty, "v2i64"); 5111c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0); 5112c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1); 5113651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0"); 5114651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1"); 5115651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Pairwise addition of a v2f64 into a scalar f64. 5116651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Op0, Op1, "vpaddd"); 5117651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5118651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpaddd_f64: { 5119651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty = 512087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::VectorType::get(DoubleTy, 2); 5121651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Vec = EmitScalarExpr(E->getArg(0)); 5122651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The vector is v2f64, so make sure it's bitcast to that. 5123651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Vec = Builder.CreateBitCast(Vec, Ty, "v2f64"); 5124c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0); 5125c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1); 5126651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0"); 5127651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1"); 5128651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Pairwise addition of a v2f64 into a scalar f64. 5129651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFAdd(Op0, Op1, "vpaddd"); 5130651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5131651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpadds_f32: { 5132651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Ty = 513387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::VectorType::get(FloatTy, 2); 5134651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Vec = EmitScalarExpr(E->getArg(0)); 5135651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The vector is v2f32, so make sure it's bitcast to that. 5136651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Vec = Builder.CreateBitCast(Vec, Ty, "v2f32"); 5137c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0); 5138c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1); 5139651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0"); 5140651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1"); 5141651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Pairwise addition of a v2f32 into a scalar f32. 5142651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFAdd(Op0, Op1, "vpaddd"); 5143651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5144651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqzd_s64: 5145651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqzd_f64: 5146651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqzs_f32: 5147651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5148651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr( 51490e2c34f92f00628d48968dfea096d36381f494cbStephen Hines Ops[0], ConvertType(E->getCallReturnType(getContext())), 51500e2c34f92f00628d48968dfea096d36381f494cbStephen Hines ICmpInst::FCMP_OEQ, ICmpInst::ICMP_EQ, "vceqz"); 5151651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgezd_s64: 5152651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgezd_f64: 5153651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgezs_f32: 5154651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5155651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr( 51560e2c34f92f00628d48968dfea096d36381f494cbStephen Hines Ops[0], ConvertType(E->getCallReturnType(getContext())), 51570e2c34f92f00628d48968dfea096d36381f494cbStephen Hines ICmpInst::FCMP_OGE, ICmpInst::ICMP_SGE, "vcgez"); 5158651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclezd_s64: 5159651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclezd_f64: 5160651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclezs_f32: 5161651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5162651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr( 51630e2c34f92f00628d48968dfea096d36381f494cbStephen Hines Ops[0], ConvertType(E->getCallReturnType(getContext())), 51640e2c34f92f00628d48968dfea096d36381f494cbStephen Hines ICmpInst::FCMP_OLE, ICmpInst::ICMP_SLE, "vclez"); 5165651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtzd_s64: 5166651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtzd_f64: 5167651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtzs_f32: 5168651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5169651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr( 51700e2c34f92f00628d48968dfea096d36381f494cbStephen Hines Ops[0], ConvertType(E->getCallReturnType(getContext())), 51710e2c34f92f00628d48968dfea096d36381f494cbStephen Hines ICmpInst::FCMP_OGT, ICmpInst::ICMP_SGT, "vcgtz"); 5172651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltzd_s64: 5173651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltzd_f64: 5174651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltzs_f32: 5175651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5176651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr( 51770e2c34f92f00628d48968dfea096d36381f494cbStephen Hines Ops[0], ConvertType(E->getCallReturnType(getContext())), 51780e2c34f92f00628d48968dfea096d36381f494cbStephen Hines ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT, "vcltz"); 5179651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 5180651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqzd_u64: { 5181651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 518287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); 518387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[0] = 518487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(Int64Ty)); 518587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateSExt(Ops[0], Int64Ty, "vceqzd"); 5186651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5187651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqd_f64: 5188651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_f64: 5189651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_f64: 5190651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcged_f64: 5191651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtd_f64: { 5192651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::CmpInst::Predicate P; 5193651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 5194651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines default: llvm_unreachable("missing builtin ID in switch!"); 5195651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqd_f64: P = llvm::FCmpInst::FCMP_OEQ; break; 5196651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_f64: P = llvm::FCmpInst::FCMP_OLE; break; 5197651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_f64: P = llvm::FCmpInst::FCMP_OLT; break; 5198651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcged_f64: P = llvm::FCmpInst::FCMP_OGE; break; 5199651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtd_f64: P = llvm::FCmpInst::FCMP_OGT; break; 5200651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5201651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 5202651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 5203651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); 5204651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); 5205651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd"); 5206651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5207651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqs_f32: 5208651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcles_f32: 5209651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclts_f32: 5210651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcges_f32: 5211651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgts_f32: { 5212651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::CmpInst::Predicate P; 5213651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 5214651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines default: llvm_unreachable("missing builtin ID in switch!"); 5215651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqs_f32: P = llvm::FCmpInst::FCMP_OEQ; break; 5216651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcles_f32: P = llvm::FCmpInst::FCMP_OLE; break; 5217651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclts_f32: P = llvm::FCmpInst::FCMP_OLT; break; 5218651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcges_f32: P = llvm::FCmpInst::FCMP_OGE; break; 5219651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgts_f32: P = llvm::FCmpInst::FCMP_OGT; break; 5220651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5221651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 5222651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy); 5223651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy); 5224651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); 5225651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd"); 5226651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5227651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqd_s64: 5228651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqd_u64: 5229651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtd_s64: 5230651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtd_u64: 5231651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_s64: 5232651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_u64: 5233651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcged_u64: 5234651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcged_s64: 5235651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_u64: 5236651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_s64: { 5237651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::CmpInst::Predicate P; 5238651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 5239651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines default: llvm_unreachable("missing builtin ID in switch!"); 5240651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqd_s64: 5241651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqd_u64:P = llvm::ICmpInst::ICMP_EQ;break; 5242651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtd_s64:P = llvm::ICmpInst::ICMP_SGT;break; 5243651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtd_u64:P = llvm::ICmpInst::ICMP_UGT;break; 5244651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_s64:P = llvm::ICmpInst::ICMP_SLT;break; 5245651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltd_u64:P = llvm::ICmpInst::ICMP_ULT;break; 5246651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcged_u64:P = llvm::ICmpInst::ICMP_UGE;break; 5247651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcged_s64:P = llvm::ICmpInst::ICMP_SGE;break; 5248651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_u64:P = llvm::ICmpInst::ICMP_ULE;break; 5249651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcled_s64:P = llvm::ICmpInst::ICMP_SLE;break; 5250651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5251651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 5252651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); 5253651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); 5254651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]); 5255651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateSExt(Ops[0], Int64Ty, "vceqd"); 5256651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5257651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtstd_s64: 5258651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtstd_u64: { 5259651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 526087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); 526187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); 5262651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); 5263651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], 526487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Constant::getNullValue(Int64Ty)); 526587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateSExt(Ops[0], Int64Ty, "vtstd"); 5266651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5267651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_i8: 5268651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_i16: 5269651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_i32: 5270651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_i64: 5271651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_f32: 5272651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_i8: 5273651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_i16: 5274651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_i32: 5275651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_i64: 5276651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_f32: 5277651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(2))); 5278651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 5279651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vset_lane_f64: 5280651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The vector type needs a cast for the v1f64 variant. 5281651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], 5282651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(DoubleTy, 1)); 5283651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(2))); 5284651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 5285651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsetq_lane_f64: 5286651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The vector type needs a cast for the v2f64 variant. 5287651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], 528887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::VectorType::get(DoubleTy, 2)); 528983bbba1fc98dd3b2a351013a40808e995e648f40Bob Wilson Ops.push_back(EmitScalarExpr(E->getArg(2))); 529083bbba1fc98dd3b2a351013a40808e995e648f40Bob Wilson return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 529199c40bb13b523d58d7aeb6446e4f486d6918ca58Nate Begeman 5292651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_i8: 5293651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdupb_lane_i8: 529487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 8)); 5295651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5296651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vget_lane"); 5297651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_i8: 5298651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdupb_laneq_i8: 529987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 16)); 5300651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5301651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vgetq_lane"); 5302651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_i16: 5303651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vduph_lane_i16: 530487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 4)); 5305651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5306651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vget_lane"); 5307651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_i16: 5308651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vduph_laneq_i16: 530987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 8)); 5310651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5311651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vgetq_lane"); 5312651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_i32: 5313651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdups_lane_i32: 531487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 2)); 5315651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5316651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vget_lane"); 5317651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdups_lane_f32: 5318651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 531987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::VectorType::get(FloatTy, 2)); 5320651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5321651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vdups_lane"); 5322651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_i32: 5323651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdups_laneq_i32: 532487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 4)); 5325651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5326651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vgetq_lane"); 5327651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_i64: 5328651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdupd_lane_i64: 532987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 1)); 5330651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5331651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vget_lane"); 5332651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdupd_lane_f64: 5333651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 533487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::VectorType::get(DoubleTy, 1)); 5335651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5336651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vdupd_lane"); 5337651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_i64: 5338651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdupd_laneq_i64: 533987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2)); 5340651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5341651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vgetq_lane"); 5342651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_f32: 5343651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 534487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::VectorType::get(FloatTy, 2)); 5345651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5346651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vget_lane"); 5347651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vget_lane_f64: 5348651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 534987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::VectorType::get(DoubleTy, 1)); 5350651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5351651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vget_lane"); 5352651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_f32: 5353651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdups_laneq_f32: 5354651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 535587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::VectorType::get(FloatTy, 4)); 5356651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5357651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vgetq_lane"); 5358651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vgetq_lane_f64: 5359651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vdupd_laneq_f64: 5360651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 536187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::VectorType::get(DoubleTy, 2)); 5362651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5363651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "vgetq_lane"); 5364651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddd_s64: 5365651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddd_u64: 5366651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->getArg(1)), "vaddd"); 5367651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsubd_s64: 5368651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsubd_u64: 5369651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateSub(Ops[0], EmitScalarExpr(E->getArg(1)), "vsubd"); 5370651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlalh_s16: 5371651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlslh_s16: { 5372651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value *, 2> ProductOps; 5373651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps.push_back(vectorWrapScalar16(Ops[1])); 5374651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps.push_back(vectorWrapScalar16(EmitScalarExpr(E->getArg(2)))); 5375651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4); 53766bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), 5377651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps, "vqdmlXl"); 5378c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Constant *CI = ConstantInt::get(SizeTy, 0); 5379651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); 5380651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 5381651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16 53826bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ? Intrinsic::aarch64_neon_sqadd 53836bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines : Intrinsic::aarch64_neon_sqsub; 5384651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops, "vqdmlXl"); 5385651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5386651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshlud_n_s64: { 5387651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 5388651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); 53896bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqshlu, Int64Ty), 53906bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops, "vqshlu_n"); 5391651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5392651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshld_n_u64: 5393651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshld_n_s64: { 5394651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64 53956bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ? Intrinsic::aarch64_neon_uqshl 53966bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines : Intrinsic::aarch64_neon_sqshl; 5397651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 5398651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); 53996bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vqshl_n"); 5400651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5401651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrshrd_n_u64: 5402651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrshrd_n_s64: { 5403651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64 54046bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ? Intrinsic::aarch64_neon_urshl 54056bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines : Intrinsic::aarch64_neon_srshl; 5406651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 54076bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines int SV = cast<ConstantInt>(Ops[1])->getSExtValue(); 54086bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[1] = ConstantInt::get(Int64Ty, -SV); 54096bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vrshr_n"); 5410651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5411651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrsrad_n_u64: 5412651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrsrad_n_s64: { 5413651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64 54146bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ? Intrinsic::aarch64_neon_urshl 54156bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines : Intrinsic::aarch64_neon_srshl; 5416651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); 5417651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2)))); 5418b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Int64Ty), 5419b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)}); 5420651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty)); 5421651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5422651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshld_n_s64: 5423651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshld_n_u64: { 5424651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1))); 5425651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateShl( 54266bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()), "shld_n"); 5427651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5428651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshrd_n_s64: { 5429651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1))); 5430651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAShr( 5431651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), 5432651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Amt->getZExtValue())), 54336bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines "shrd_n"); 5434651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5435651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vshrd_n_u64: { 5436651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1))); 54376bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines uint64_t ShiftAmt = Amt->getZExtValue(); 54386bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // Right-shifting an unsigned value by its size yields 0. 54396bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (ShiftAmt == 64) 54406bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return ConstantInt::get(Int64Ty, 0); 54416bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt), 54426bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines "shrd_n"); 5443651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5444651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsrad_n_s64: { 5445651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2))); 5446651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateAShr( 5447651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), 5448651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Amt->getZExtValue())), 54496bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines "shrd_n"); 5450651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Ops[0], Ops[1]); 5451651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5452651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsrad_n_u64: { 5453651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2))); 54546bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines uint64_t ShiftAmt = Amt->getZExtValue(); 54556bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // Right-shifting an unsigned value by its size yields 0. 54566bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // As Op + 0 = Op, return Ops[0] directly. 54576bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (ShiftAmt == 64) 54586bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return Ops[0]; 54596bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt), 54606bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines "shrd_n"); 5461651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Ops[0], Ops[1]); 5462651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5463651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlalh_lane_s16: 5464651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlalh_laneq_s16: 5465651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlslh_lane_s16: 5466651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: { 5467651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), 5468651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "lane"); 5469651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value *, 2> ProductOps; 5470651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps.push_back(vectorWrapScalar16(Ops[1])); 5471651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps.push_back(vectorWrapScalar16(Ops[2])); 5472651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4); 54736bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), 5474651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps, "vqdmlXl"); 5475c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Constant *CI = ConstantInt::get(SizeTy, 0); 5476651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); 5477651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.pop_back(); 5478651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 5479651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 || 5480651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16) 54816bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ? Intrinsic::aarch64_neon_sqadd 54826bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines : Intrinsic::aarch64_neon_sqsub; 5483651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops, "vqdmlXl"); 5484651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5485651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlals_s32: 5486651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlsls_s32: { 5487651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value *, 2> ProductOps; 5488651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps.push_back(Ops[1]); 5489651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps.push_back(EmitScalarExpr(E->getArg(2))); 5490651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = 54916bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar), 5492651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps, "vqdmlXl"); 5493651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 5494651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32 54956bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ? Intrinsic::aarch64_neon_sqadd 54966bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines : Intrinsic::aarch64_neon_sqsub; 5497651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops, "vqdmlXl"); 5498651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5499651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlals_lane_s32: 5500651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlals_laneq_s32: 5501651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlsls_lane_s32: 5502651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: { 5503651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), 5504651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "lane"); 5505651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<Value *, 2> ProductOps; 5506651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps.push_back(Ops[1]); 5507651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps.push_back(Ops[2]); 5508651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = 55096bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar), 5510651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ProductOps, "vqdmlXl"); 5511651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.pop_back(); 5512651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 5513651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 || 5514651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32) 55156bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ? Intrinsic::aarch64_neon_sqadd 55166bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines : Intrinsic::aarch64_neon_sqsub; 5517651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl"); 5518651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 551999c40bb13b523d58d7aeb6446e4f486d6918ca58Nate Begeman } 5520e140af3e27016f902146023fba7680b43043ec07Rafael Espindola 55218b418685e9e4f02f4eb2a76e1ec063e07552b68dChris Lattner llvm::VectorType *VTy = GetNeonType(this, Type); 55229cbe4f0ba01ec304e1e3d071c071f7bca33631c0Chris Lattner llvm::Type *Ty = VTy; 5523e140af3e27016f902146023fba7680b43043ec07Rafael Espindola if (!Ty) 55246bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return nullptr; 5525e140af3e27016f902146023fba7680b43043ec07Rafael Espindola 55266bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines // Not all intrinsics handled by the common case work for AArch64 yet, so only 5527651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // defer to common code if it's been added to our special map. 55286bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Builtin = findNeonIntrinsicInMap(AArch64SIMDIntrinsicMap, BuiltinID, 55296bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines AArch64SIMDIntrinsicsProvenSorted); 5530651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 5531651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (Builtin) 5532651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitCommonNeonBuiltinExpr( 5533651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic, 553487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builtin->NameHint, Builtin->TypeModifier, E, Ops, 553587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar /*never use addresses*/ Address::invalid(), Address::invalid()); 5536651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 55376bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops)) 5538651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return V; 5539651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 5540e140af3e27016f902146023fba7680b43043ec07Rafael Espindola unsigned Int; 5541e140af3e27016f902146023fba7680b43043ec07Rafael Espindola switch (BuiltinID) { 55426bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines default: return nullptr; 5543651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vbsl_v: 5544651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vbslq_v: { 5545651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *BitTy = llvm::VectorType::getInteger(VTy); 5546651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl"); 5547651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl"); 5548651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl"); 5549651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 5550651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl"); 5551651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl"); 5552651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl"); 5553651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateBitCast(Ops[0], Ty); 5554651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5555651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfma_lane_v: 5556651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfmaq_lane_v: { // Only used for FP types 5557651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // The ARM builtins (and instructions) have the addend as the first 5558651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // operand, but the 'fma' intrinsics have it last. Swap it around here. 5559651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Addend = Ops[0]; 5560651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Multiplicand = Ops[1]; 5561651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *LaneSource = Ops[2]; 5562651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Multiplicand; 5563651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = LaneSource; 5564651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Addend; 5565651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 5566651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Now adjust things to handle the lane access. 5567651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v ? 5568651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType::get(VTy->getElementType(), VTy->getNumElements() / 2) : 5569651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy; 5570651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Constant *cst = cast<Constant>(Ops[3]); 5571651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *SV = llvm::ConstantVector::getSplat(VTy->getNumElements(), cst); 5572651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy); 5573651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane"); 5574651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 5575651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.pop_back(); 5576651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Intrinsic::fma; 5577651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmla"); 5578651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5579651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfma_laneq_v: { 5580651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::VectorType *VTy = cast<llvm::VectorType>(Ty); 5581651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // v1f64 fma should be mapped to Neon scalar f64 fma 5582651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (VTy && VTy->getElementType() == DoubleTy) { 5583651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 5584651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); 5585651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *VTy = GetNeonType(this, 5586651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags(NeonTypeFlags::Float64, false, true)); 5587651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateBitCast(Ops[2], VTy); 5588651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); 5589651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *F = CGM.getIntrinsic(Intrinsic::fma, DoubleTy); 5590b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *Result = Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); 5591651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateBitCast(Result, Ty); 5592651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5593651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 5594651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5595651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 55966bf1e8eadd36f077cf5403d23c664129d98eaf79Tim Northover 5597651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *STy = llvm::VectorType::get(VTy->getElementType(), 5598651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines VTy->getNumElements() * 2); 5599651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateBitCast(Ops[2], STy); 5600651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value* SV = llvm::ConstantVector::getSplat(VTy->getNumElements(), 5601651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines cast<ConstantInt>(Ops[3])); 5602651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane"); 56036bf1e8eadd36f077cf5403d23c664129d98eaf79Tim Northover 5604b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]}); 5605651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5606651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfmaq_laneq_v: { 5607651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 5608651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5609651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 56106bf1e8eadd36f077cf5403d23c664129d98eaf79Tim Northover 5611651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 5612651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3])); 5613b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]}); 56146bf1e8eadd36f077cf5403d23c664129d98eaf79Tim Northover } 5615651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfmas_lane_f32: 5616651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfmas_laneq_f32: 5617651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfmad_lane_f64: 5618651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vfmad_laneq_f64: { 5619651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(3))); 56200e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext())); 5621651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 5622651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); 5623b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); 562430d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman } 5625651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmull_v: 5626651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 56276bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull; 56286bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (Type.isPoly()) Int = Intrinsic::aarch64_neon_pmull; 5629651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); 5630651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmax_v: 5631651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxq_v: 5632651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 56336bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax; 56346bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax; 5635651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax"); 5636651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmin_v: 5637651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminq_v: 5638651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 56396bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin; 56406bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin; 5641651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin"); 5642651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vabd_v: 5643651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vabdq_v: 5644651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 56456bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd; 56466bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd; 5647651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd"); 5648651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpadal_v: 5649651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpadalq_v: { 5650651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned ArgElts = VTy->getNumElements(); 5651651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType()); 5652651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned BitWidth = EltTy->getBitWidth(); 5653651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *ArgTy = llvm::VectorType::get( 5654651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::IntegerType::get(getLLVMContext(), BitWidth/2), 2*ArgElts); 5655651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type* Tys[2] = { VTy, ArgTy }; 56566bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp; 5657651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<llvm::Value*, 1> TmpOps; 5658651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TmpOps.push_back(Ops[1]); 5659651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(Int, Tys); 5660651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vpadal"); 5661651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType()); 5662651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(tmp, addend); 5663651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5664651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpmin_v: 5665651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpminq_v: 5666651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 56676bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp; 56686bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp; 5669651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin"); 5670651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpmax_v: 5671651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpmaxq_v: 5672651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 56736bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp; 56746bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp; 5675651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax"); 5676651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminnm_v: 5677651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminnmq_v: 56786bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_fminnm; 5679651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vminnm"); 5680651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxnm_v: 5681651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxnmq_v: 56826bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_fmaxnm; 5683651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm"); 5684651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrecpss_f32: { 5685651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 568687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, FloatTy), 5687651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vrecps"); 568830d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman } 5689651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrecpsd_f64: { 5690651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(1))); 569187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, DoubleTy), 5692651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vrecps"); 5693651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5694651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshrun_n_v: 56956bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_sqshrun; 5696651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrun_n"); 5697651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqrshrun_n_v: 56986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_sqrshrun; 5699651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrun_n"); 5700651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqshrn_n_v: 57016bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn; 5702651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n"); 5703651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrshrn_n_v: 57046bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_rshrn; 5705651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshrn_n"); 5706651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqrshrn_n_v: 57076bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn; 5708651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n"); 5709651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrnda_v: 5710651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndaq_v: { 5711651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Intrinsic::round; 5712651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnda"); 5713651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5714651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndi_v: 5715651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndiq_v: { 5716651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Intrinsic::nearbyint; 5717651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndi"); 5718651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5719651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndm_v: 5720651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndmq_v: { 5721651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Intrinsic::floor; 5722651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndm"); 5723651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5724651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndn_v: 5725651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndnq_v: { 57266bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_frintn; 5727651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn"); 5728651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5729651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndp_v: 5730651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndpq_v: { 5731651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Intrinsic::ceil; 5732651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndp"); 5733651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5734651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndx_v: 5735651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndxq_v: { 5736651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Intrinsic::rint; 5737651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndx"); 5738651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5739651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrnd_v: 5740651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrndq_v: { 5741651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Intrinsic::trunc; 5742651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndz"); 5743651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5744651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqz_v: 5745651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vceqzq_v: 5746651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OEQ, 5747651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ICmpInst::ICMP_EQ, "vceqz"); 5748651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgez_v: 5749651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgezq_v: 5750651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGE, 5751651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ICmpInst::ICMP_SGE, "vcgez"); 5752651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclez_v: 5753651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vclezq_v: 5754651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLE, 5755651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ICmpInst::ICMP_SLE, "vclez"); 5756651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtz_v: 5757651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcgtzq_v: 5758651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT, 5759651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ICmpInst::ICMP_SGT, "vcgtz"); 5760651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltz_v: 5761651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcltzq_v: 5762651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT, 5763651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ICmpInst::ICMP_SLT, "vcltz"); 5764651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_f64_v: 5765651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_f64_v: 576630d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5767651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float64, false, quad)); 5768258f930227c1a102c9c22eee88df65f748863425Jim Grosbach return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") 57699eb65a56e18bee1e5392bf2dff01cbd7b895f685Nate Begeman : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); 5770651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_f64_f32: { 5771651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines assert(Type.getEltType() == NeonTypeFlags::Float64 && quad && 5772651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "unexpected vcvt_f64_f32 builtin"); 5773651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float32, false, false); 5774651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); 5775651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 5776651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFPExt(Ops[0], Ty, "vcvt"); 57779eb65a56e18bee1e5392bf2dff01cbd7b895f685Nate Begeman } 5778651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_f32_f64: { 5779651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines assert(Type.getEltType() == NeonTypeFlags::Float32 && 5780651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines "unexpected vcvt_f32_f64 builtin"); 5781651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float64, false, true); 5782651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); 5783651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 5784651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt"); 57859eb65a56e18bee1e5392bf2dff01cbd7b895f685Nate Begeman } 5786651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_s32_v: 5787651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_u32_v: 5788651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_s64_v: 5789651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvt_u64_v: 5790651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_s32_v: 5791651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_u32_v: 5792651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_s64_v: 5793651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtq_u64_v: { 579487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type)); 5795651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (usgn) 5796651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFPToUI(Ops[0], Ty); 5797651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFPToSI(Ops[0], Ty); 5798651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5799651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvta_s32_v: 5800651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtaq_s32_v: 5801651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvta_u32_v: 5802651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtaq_u32_v: 5803651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvta_s64_v: 5804651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtaq_s64_v: 5805651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvta_u64_v: 5806651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtaq_u64_v: { 58076bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas; 580887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; 5809651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta"); 5810651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5811651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtm_s32_v: 5812651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtmq_s32_v: 5813651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtm_u32_v: 5814651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtmq_u32_v: 5815651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtm_s64_v: 5816651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtmq_s64_v: 5817651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtm_u64_v: 5818651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtmq_u64_v: { 58196bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms; 582087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; 5821651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm"); 5822651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5823651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtn_s32_v: 5824651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtnq_s32_v: 5825651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtn_u32_v: 5826651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtnq_u32_v: 5827651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtn_s64_v: 5828651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtnq_s64_v: 5829651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtn_u64_v: 5830651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtnq_u64_v: { 58316bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns; 583287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; 5833651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn"); 5834651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5835651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtp_s32_v: 5836651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtpq_s32_v: 5837651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtp_u32_v: 5838651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtpq_u32_v: 5839651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtp_s64_v: 5840651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtpq_s64_v: 5841651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtp_u64_v: 5842651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vcvtpq_u64_v: { 58436bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps; 584487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; 5845651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp"); 5846651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5847651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmulx_v: 5848651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmulxq_v: { 58496bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_fmulx; 5850651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmulx"); 585130d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman } 5852651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmul_lane_v: 5853651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmul_laneq_v: { 5854651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // v1f64 vmul_lane should be mapped to Neon scalar mul lane 5855651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool Quad = false; 5856651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v) 5857651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Quad = true; 5858651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 5859651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *VTy = GetNeonType(this, 5860651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NeonTypeFlags(NeonTypeFlags::Float64, false, Quad)); 5861651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], VTy); 5862651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract"); 5863651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Result = Builder.CreateFMul(Ops[0], Ops[1]); 5864651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateBitCast(Result, Ty); 5865651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5866651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vnegd_s64: 5867651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateNeg(EmitScalarExpr(E->getArg(0)), "vnegd"); 5868651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpmaxnm_v: 5869651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpmaxnmq_v: { 58706bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_fmaxnmp; 5871651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmaxnm"); 5872651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5873651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpminnm_v: 5874651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vpminnmq_v: { 58756bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_fminnmp; 5876651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpminnm"); 5877651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5878651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsqrt_v: 5879651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsqrtq_v: { 5880651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Int = Intrinsic::sqrt; 588130d91718a676177f0d0d0210ce4fdb4f616df6e5Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5882651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqrt"); 58831c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman } 5884651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrbit_v: 5885651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrbitq_v: { 58866bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_rbit; 5887651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrbit"); 5888651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5889651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddv_u8: 5890651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: These are handled by the AArch64 scalar code. 5891651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines usgn = true; 5892651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FALLTHROUGH 5893651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddv_s8: { 58946bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 589587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 589687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int8Ty, 8); 5897651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5898651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5899651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 590087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateTrunc(Ops[0], Int8Ty); 5901651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5902651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddv_u16: 5903651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines usgn = true; 5904651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FALLTHROUGH 5905651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddv_s16: { 59066bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 590787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 590887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int16Ty, 4); 5909651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5910651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5911651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 591287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateTrunc(Ops[0], Int16Ty); 5913651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5914651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddvq_u8: 5915651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines usgn = true; 5916651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FALLTHROUGH 5917651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddvq_s8: { 59186bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 591987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 592087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int8Ty, 16); 5921651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5922651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5923651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 592487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateTrunc(Ops[0], Int8Ty); 5925651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5926651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddvq_u16: 5927651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines usgn = true; 5928651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FALLTHROUGH 5929651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddvq_s16: { 59306bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 593187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 593287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int16Ty, 8); 5933651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5934651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5935651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 593687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateTrunc(Ops[0], Int16Ty); 5937651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5938651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxv_u8: { 59396bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_umaxv; 594087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 594187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int8Ty, 8); 5942651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5943651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5944651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 594587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateTrunc(Ops[0], Int8Ty); 5946651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5947651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxv_u16: { 59486bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_umaxv; 594987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 595087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int16Ty, 4); 5951651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5952651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5953651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 595487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateTrunc(Ops[0], Int16Ty); 5955651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5956651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxvq_u8: { 59576bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_umaxv; 595887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 595987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int8Ty, 16); 5960651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5961651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5962651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 596387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateTrunc(Ops[0], Int8Ty); 5964651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5965651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxvq_u16: { 59666bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_umaxv; 596787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 596887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int16Ty, 8); 5969651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5970651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5971651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 597287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateTrunc(Ops[0], Int16Ty); 5973651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5974651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxv_s8: { 59756bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_smaxv; 597687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 597787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int8Ty, 8); 5978651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5979651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5980651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 598187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateTrunc(Ops[0], Int8Ty); 5982651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5983651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxv_s16: { 59846bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_smaxv; 598587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 598687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int16Ty, 4); 5987651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5988651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5989651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 599087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateTrunc(Ops[0], Int16Ty); 5991651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 5992651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxvq_s8: { 59936bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_smaxv; 599487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 599587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int8Ty, 16); 5996651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 5997651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 5998651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 599987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateTrunc(Ops[0], Int8Ty); 6000651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6001651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmaxvq_s16: { 60026bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_smaxv; 600387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 600487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int16Ty, 8); 6005651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 6006651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 6007651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 600887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateTrunc(Ops[0], Int16Ty); 6009651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6010651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminv_u8: { 60116bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_uminv; 601287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 601387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int8Ty, 8); 6014651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 6015651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 6016651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 601787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateTrunc(Ops[0], Int8Ty); 6018651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6019651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminv_u16: { 60206bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_uminv; 602187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 602287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int16Ty, 4); 6023651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 6024651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 6025651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 602687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateTrunc(Ops[0], Int16Ty); 6027651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6028651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminvq_u8: { 60296bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_uminv; 603087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 603187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int8Ty, 16); 6032651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 6033651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 6034651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 603587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateTrunc(Ops[0], Int8Ty); 6036651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6037651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminvq_u16: { 60386bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_uminv; 603987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 604087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int16Ty, 8); 6041651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 6042651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 6043651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 604487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateTrunc(Ops[0], Int16Ty); 6045651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6046651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminv_s8: { 60476bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_sminv; 604887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 604987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int8Ty, 8); 6050651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 6051651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 6052651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 605387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateTrunc(Ops[0], Int8Ty); 6054651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6055651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminv_s16: { 60566bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_sminv; 605787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 605887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int16Ty, 4); 6059651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 6060651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 6061651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 606287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateTrunc(Ops[0], Int16Ty); 6063651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6064651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminvq_s8: { 60656bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_sminv; 606687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 606787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int8Ty, 16); 6068651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 6069651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 6070651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 607187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateTrunc(Ops[0], Int8Ty); 6072651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6073651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vminvq_s16: { 60746bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_sminv; 607587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 607687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int16Ty, 8); 6077651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 6078651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 6079651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 608087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateTrunc(Ops[0], Int16Ty); 6081651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6082651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vmul_n_f64: { 6083651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 6084651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *RHS = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), DoubleTy); 6085651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateFMul(Ops[0], RHS); 6086651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6087651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddlv_u8: { 60886bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_uaddlv; 608987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 609087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int8Ty, 8); 6091651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 6092651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 6093651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 609487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateTrunc(Ops[0], Int16Ty); 6095651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6096651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddlv_u16: { 60976bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_uaddlv; 609887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 609987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int16Ty, 4); 6100651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 6101651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 6102651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 6103651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6104651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddlvq_u8: { 61056bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_uaddlv; 610687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 610787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int8Ty, 16); 6108651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 6109651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 6110651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 611187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateTrunc(Ops[0], Int16Ty); 6112651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6113651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddlvq_u16: { 61146bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_uaddlv; 611587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 611687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int16Ty, 8); 6117651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 6118651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 6119651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 6120651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6121651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddlv_s8: { 61226bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_saddlv; 612387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 612487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int8Ty, 8); 6125651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 6126651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 6127651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 612887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateTrunc(Ops[0], Int16Ty); 6129651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6130651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddlv_s16: { 61316bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_saddlv; 613287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 613387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int16Ty, 4); 6134651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 6135651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 6136651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 6137651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6138651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddlvq_s8: { 61396bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_saddlv; 614087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 614187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int8Ty, 16); 6142651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 6143651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 6144651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 614587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateTrunc(Ops[0], Int16Ty); 6146651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6147651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vaddlvq_s16: { 61486bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_saddlv; 614987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ty = Int32Ty; 615087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar VTy = llvm::VectorType::get(Int16Ty, 8); 6151651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { Ty, VTy }; 6152651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(EmitScalarExpr(E->getArg(0))); 6153651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 6154651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6155651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsri_n_v: 6156651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsriq_n_v: { 61576bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_vsri; 6158651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty); 6159651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(Intrin, Ops, "vsri_n"); 6160651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6161651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsli_n_v: 6162651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsliq_n_v: { 61636bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_vsli; 6164651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty); 6165651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(Intrin, Ops, "vsli_n"); 6166651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6167651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsra_n_v: 6168651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsraq_n_v: 6169651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 6170651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); 6171651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Ops[0], Ops[1]); 6172651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrsra_n_v: 6173651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vrsraq_n_v: { 61746bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl; 6175651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SmallVector<llvm::Value*,2> TmpOps; 6176651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TmpOps.push_back(Ops[1]); 6177651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines TmpOps.push_back(Ops[2]); 6178651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function* F = CGM.getIntrinsic(Int, Ty); 6179651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vrshr_n", 1, true); 6180651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], VTy); 6181651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateAdd(Ops[0], tmp); 6182651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6183651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // FIXME: Sharing loads & stores with 32-bit is complicated by the absence 6184651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // of an Align parameter here. 6185651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_x2_v: 6186651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_x2_v: 6187651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_x3_v: 6188651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_x3_v: 6189651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_x4_v: 6190651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_x4_v: { 6191651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType()); 6192651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 6193651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, PTy }; 6194651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Int; 6195651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 6196651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_x2_v: 6197651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_x2_v: 61986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_ld1x2; 6199651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 6200651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_x3_v: 6201651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_x3_v: 62026bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_ld1x3; 6203651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 6204651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_x4_v: 6205651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_x4_v: 62066bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_ld1x4; 6207651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 6208550a9d823a939366a9f776b58f18883acd905a93Bob Wilson } 6209651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Function *F = CGM.getIntrinsic(Int, Tys); 6210651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN"); 6211651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 6212651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 621387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6214651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6215651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_x2_v: 6216651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_x2_v: 6217651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_x3_v: 6218651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_x3_v: 6219651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_x4_v: 6220651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_x4_v: { 6221651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType()); 6222651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, PTy }; 6223651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines unsigned Int; 6224651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (BuiltinID) { 6225651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_x2_v: 6226651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_x2_v: 62276bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_st1x2; 6228651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 6229651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_x3_v: 6230651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_x3_v: 62316bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_st1x3; 6232651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 6233651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_x4_v: 6234651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_x4_v: 62356bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_st1x4; 6236651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 6237651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 623887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); 623987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, ""); 6240651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6241651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_v: 6242651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_v: 6243651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); 624487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateDefaultAlignedLoad(Ops[0]); 6245651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_v: 6246651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_v: 6247651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); 6248651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], VTy); 624987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6250651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_lane_v: 6251651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_lane_v: 62524be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 62534be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ty = llvm::PointerType::getUnqual(VTy->getElementType()); 62544be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 625587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[0] = Builder.CreateDefaultAlignedLoad(Ops[0]); 6256651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane"); 6257651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1_dup_v: 6258651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld1q_dup_v: { 62594be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Value *V = UndefValue::get(Ty); 62604be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ty = llvm::PointerType::getUnqual(VTy->getElementType()); 62614be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 626287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[0] = Builder.CreateDefaultAlignedLoad(Ops[0]); 626377b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner llvm::Constant *CI = ConstantInt::get(Int32Ty, 0); 6264651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI); 62654be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman return EmitNeonSplat(Ops[0], CI); 62664be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 6267651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1_lane_v: 6268651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst1q_lane_v: 6269651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 6270651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); 62714be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 627287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateDefaultAlignedStore(Ops[1], 627387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateBitCast(Ops[0], Ty)); 6274651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_v: 6275651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2q_v: { 6276651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *PTy = llvm::PointerType::getUnqual(VTy); 6277651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 6278651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, PTy }; 62796bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2, Tys); 6280651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); 6281651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 6282651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(Ops[1]->getType())); 628387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 62844be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 6285651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_v: 6286651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3q_v: { 6287651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *PTy = llvm::PointerType::getUnqual(VTy); 6288651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 6289651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, PTy }; 62906bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3, Tys); 6291651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); 6292651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 6293651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(Ops[1]->getType())); 629487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 62954be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 6296651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_v: 6297651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4q_v: { 6298651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *PTy = llvm::PointerType::getUnqual(VTy); 6299651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 6300651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, PTy }; 63016bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4, Tys); 6302651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); 6303651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 6304651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(Ops[1]->getType())); 630587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6306651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6307651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_dup_v: 6308651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2q_dup_v: { 6309651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *PTy = 6310651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(VTy->getElementType()); 6311651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 6312651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, PTy }; 63136bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2r, Tys); 6314651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); 6315651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 6316651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(Ops[1]->getType())); 631787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 63184be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 6319651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_dup_v: 6320651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3q_dup_v: { 6321651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *PTy = 6322651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(VTy->getElementType()); 6323651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 6324651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, PTy }; 63256bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3r, Tys); 6326651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); 6327651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 6328651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(Ops[1]->getType())); 632987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6330651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6331651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_dup_v: 6332651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4q_dup_v: { 6333651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *PTy = 6334651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(VTy->getElementType()); 6335651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 6336651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, PTy }; 63376bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4r, Tys); 6338651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); 6339651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[0] = Builder.CreateBitCast(Ops[0], 6340651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::PointerType::getUnqual(Ops[1]->getType())); 634187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6342651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6343651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2_lane_v: 6344651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld2q_lane_v: { 6345651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; 63466bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2lane, Tys); 6347651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[1]); 6348651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()+1); 6349651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 63504be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 635187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); 6352176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld2_lane"); 63534be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 63544be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 635587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 63564be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 6357651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3_lane_v: 6358651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld3q_lane_v: { 6359651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; 63606bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3lane, Tys); 6361651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[1]); 6362651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()+1); 6363651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 63644be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 63654be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[3] = Builder.CreateBitCast(Ops[3], Ty); 636687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty); 6367176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane"); 63684be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 63694be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 637087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 63714be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 6372651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4_lane_v: 6373651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vld4q_lane_v: { 6374651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; 63756bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4lane, Tys); 6376651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[1]); 6377651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()+1); 6378651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 63794be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 63804be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[3] = Builder.CreateBitCast(Ops[3], Ty); 63814be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[4] = Builder.CreateBitCast(Ops[4], Ty); 638287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[5] = Builder.CreateZExt(Ops[5], Int64Ty); 6383176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld4_lane"); 63844be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 63854be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 638687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 63874be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 6388651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2_v: 6389651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2q_v: { 6390651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[0]); 6391651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()); 6392651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[2]->getType() }; 63936bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2, Tys), 6394464ccb68f22a7e1c0a2844551c16f721540c91c3Nate Begeman Ops, ""); 6395eac1f6746a2915fea3ed42284b07e274c0095a95Bob Wilson } 6396651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2_lane_v: 6397651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst2q_lane_v: { 6398651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[0]); 6399651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()); 640087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty); 6401651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; 64026bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2lane, Tys), 6403464ccb68f22a7e1c0a2844551c16f721540c91c3Nate Begeman Ops, ""); 6404651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6405651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3_v: 6406651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3q_v: { 6407651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[0]); 6408651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()); 6409651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; 64106bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3, Tys), 6411464ccb68f22a7e1c0a2844551c16f721540c91c3Nate Begeman Ops, ""); 6412651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6413651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3_lane_v: 6414651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst3q_lane_v: { 6415651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[0]); 6416651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()); 641787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); 6418651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; 64196bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3lane, Tys), 6420464ccb68f22a7e1c0a2844551c16f721540c91c3Nate Begeman Ops, ""); 6421651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6422651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4_v: 6423651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4q_v: { 6424651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[0]); 6425651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()); 6426651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; 64276bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4, Tys), 6428464ccb68f22a7e1c0a2844551c16f721540c91c3Nate Begeman Ops, ""); 64296bf1e8eadd36f077cf5403d23c664129d98eaf79Tim Northover } 6430651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4_lane_v: 6431651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vst4q_lane_v: { 6432651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.push_back(Ops[0]); 6433651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops.erase(Ops.begin()); 643487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty); 6435651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Type *Tys[2] = { VTy, Ops[5]->getType() }; 64366bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4lane, Tys), 6437651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, ""); 64381c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman } 6439651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtrn_v: 6440651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vtrnq_v: { 64414be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 64424be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 64434be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 64446bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *SV = nullptr; 64454be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman 64461c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman for (unsigned vi = 0; vi != 2; ++vi) { 64474967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SmallVector<uint32_t, 16> Indices; 64484be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 64494967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Indices.push_back(i+vi); 64504967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Indices.push_back(i+e+vi); 64511c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman } 645258878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); 64534967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn"); 645487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar SV = Builder.CreateDefaultAlignedStore(SV, Addr); 64551c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman } 64564be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman return SV; 64571c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman } 6458651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vuzp_v: 6459651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vuzpq_v: { 64604be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 64611c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 64624be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 64636bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *SV = nullptr; 6464258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 64654be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman for (unsigned vi = 0; vi != 2; ++vi) { 64664967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SmallVector<uint32_t, 16> Indices; 64674be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) 64684967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Indices.push_back(2*i+vi); 64694be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman 647058878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); 64714967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp"); 647287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar SV = Builder.CreateDefaultAlignedStore(SV, Addr); 64734be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 64744be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman return SV; 64751c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman } 6476651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vzip_v: 6477651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vzipq_v: { 64784be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 64791c2a88cfaeb11227d3a6bf7204207e0c8cf6de6fNate Begeman Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 64804be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 64816bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Value *SV = nullptr; 6482258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 64834be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman for (unsigned vi = 0; vi != 2; ++vi) { 64844967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SmallVector<uint32_t, 16> Indices; 64854be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 64864967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Indices.push_back((i + vi*e) >> 1); 64874967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Indices.push_back(((i + vi*e) >> 1)+e); 64884be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 648958878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); 64904967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip"); 649187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar SV = Builder.CreateDefaultAlignedStore(SV, Addr); 64924be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman } 64934be54302da40d3e7cba3d93115f312d2fcca1879Nate Begeman return SV; 64949eb65a56e18bee1e5392bf2dff01cbd7b895f685Nate Begeman } 6495651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl1q_v: { 64966bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl1, Ty), 6497651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbl1"); 6498651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6499651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl2q_v: { 65006bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl2, Ty), 6501651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbl2"); 6502651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6503651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl3q_v: { 65046bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl3, Ty), 6505651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbl3"); 6506651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6507651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbl4q_v: { 65086bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl4, Ty), 6509651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbl4"); 6510651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6511651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx1q_v: { 65126bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx1, Ty), 6513651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbx1"); 6514651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6515651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx2q_v: { 65166bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx2, Ty), 6517651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbx2"); 6518651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6519651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx3q_v: { 65206bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx3, Ty), 6521651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbx3"); 6522651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6523651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vqtbx4q_v: { 65246bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx4, Ty), 6525651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ops, "vtbx4"); 6526651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6527651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsqadd_v: 6528651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vsqaddq_v: { 65296bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_usqadd; 6530651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqadd"); 6531651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 6532651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vuqadd_v: 6533651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case NEON::BI__builtin_neon_vuqaddq_v: { 65346bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Int = Intrinsic::aarch64_neon_suqadd; 6535651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vuqadd"); 6536651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 65372752c0137d95aa2f4ee1cdff4b564bac842e041bChris Lattner } 65382752c0137d95aa2f4ee1cdff4b564bac842e041bChris Lattner} 65392752c0137d95aa2f4ee1cdff4b564bac842e041bChris Lattner 6540aa51e513850688b7963efc62abf1eface7037602Bill Wendlingllvm::Value *CodeGenFunction:: 6541795b10062c2eaffae9e04241fb1a73cdbcb24a37Bill WendlingBuildVector(ArrayRef<llvm::Value*> Ops) { 6542aa51e513850688b7963efc62abf1eface7037602Bill Wendling assert((Ops.size() & (Ops.size() - 1)) == 0 && 6543aa51e513850688b7963efc62abf1eface7037602Bill Wendling "Not a power-of-two sized vector!"); 6544aa51e513850688b7963efc62abf1eface7037602Bill Wendling bool AllConstants = true; 6545aa51e513850688b7963efc62abf1eface7037602Bill Wendling for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i) 6546aa51e513850688b7963efc62abf1eface7037602Bill Wendling AllConstants &= isa<Constant>(Ops[i]); 6547aa51e513850688b7963efc62abf1eface7037602Bill Wendling 6548aa51e513850688b7963efc62abf1eface7037602Bill Wendling // If this is a constant vector, create a ConstantVector. 6549aa51e513850688b7963efc62abf1eface7037602Bill Wendling if (AllConstants) { 65502ce8842641cc312628c4be836d34eb250e7c3f78Chris Lattner SmallVector<llvm::Constant*, 16> CstOps; 6551aa51e513850688b7963efc62abf1eface7037602Bill Wendling for (unsigned i = 0, e = Ops.size(); i != e; ++i) 6552aa51e513850688b7963efc62abf1eface7037602Bill Wendling CstOps.push_back(cast<Constant>(Ops[i])); 6553aa51e513850688b7963efc62abf1eface7037602Bill Wendling return llvm::ConstantVector::get(CstOps); 6554aa51e513850688b7963efc62abf1eface7037602Bill Wendling } 6555aa51e513850688b7963efc62abf1eface7037602Bill Wendling 6556aa51e513850688b7963efc62abf1eface7037602Bill Wendling // Otherwise, insertelement the values to build the vector. 6557aa51e513850688b7963efc62abf1eface7037602Bill Wendling Value *Result = 6558aa51e513850688b7963efc62abf1eface7037602Bill Wendling llvm::UndefValue::get(llvm::VectorType::get(Ops[0]->getType(), Ops.size())); 6559aa51e513850688b7963efc62abf1eface7037602Bill Wendling 6560aa51e513850688b7963efc62abf1eface7037602Bill Wendling for (unsigned i = 0, e = Ops.size(); i != e; ++i) 65612ce8842641cc312628c4be836d34eb250e7c3f78Chris Lattner Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt32(i)); 6562aa51e513850688b7963efc62abf1eface7037602Bill Wendling 6563aa51e513850688b7963efc62abf1eface7037602Bill Wendling return Result; 6564aa51e513850688b7963efc62abf1eface7037602Bill Wendling} 6565aa51e513850688b7963efc62abf1eface7037602Bill Wendling 65664967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar// Convert the mask from an integer type to a vector of i1. 65674967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainarstatic Value *getMaskVecValue(CodeGenFunction &CGF, Value *Mask, 65684967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar unsigned NumElts) { 65694967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 65704967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::VectorType *MaskTy = llvm::VectorType::get(CGF.Builder.getInt1Ty(), 65714967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar cast<IntegerType>(Mask->getType())->getBitWidth()); 65724967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *MaskVec = CGF.Builder.CreateBitCast(Mask, MaskTy); 65734967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 65744967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // If we have less than 8 elements, then the starting mask was an i8 and 65754967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // we need to extract down to the right number of elements. 65764967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (NumElts < 8) { 65774967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar uint32_t Indices[4]; 65784967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar for (unsigned i = 0; i != NumElts; ++i) 65794967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Indices[i] = i; 65804967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar MaskVec = CGF.Builder.CreateShuffleVector(MaskVec, MaskVec, 65814967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar makeArrayRef(Indices, NumElts), 65824967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar "extract"); 65834967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 65844967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return MaskVec; 65854967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar} 65864967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 65874967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainarstatic Value *EmitX86MaskedStore(CodeGenFunction &CGF, 65884967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SmallVectorImpl<Value *> &Ops, 65894967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar unsigned Align) { 65904967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Cast the pointer to right type. 65914967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Ops[0] = CGF.Builder.CreateBitCast(Ops[0], 65924967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::PointerType::getUnqual(Ops[1]->getType())); 65934967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 65944967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // If the mask is all ones just emit a regular store. 65954967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (const auto *C = dyn_cast<Constant>(Ops[2])) 65964967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (C->isAllOnesValue()) 65974967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return CGF.Builder.CreateAlignedStore(Ops[1], Ops[0], Align); 65984967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 65994967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *MaskVec = getMaskVecValue(CGF, Ops[2], 66004967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Ops[1]->getType()->getVectorNumElements()); 66014967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 66024967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return CGF.Builder.CreateMaskedStore(Ops[1], Ops[0], Align, MaskVec); 66034967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar} 66044967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 66054967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainarstatic Value *EmitX86MaskedLoad(CodeGenFunction &CGF, 66064967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SmallVectorImpl<Value *> &Ops, unsigned Align) { 66074967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Cast the pointer to right type. 66084967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Ops[0] = CGF.Builder.CreateBitCast(Ops[0], 66094967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::PointerType::getUnqual(Ops[1]->getType())); 66104967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 66114967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // If the mask is all ones just emit a regular store. 66124967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (const auto *C = dyn_cast<Constant>(Ops[2])) 66134967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (C->isAllOnesValue()) 66144967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return CGF.Builder.CreateAlignedLoad(Ops[0], Align); 66154967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 66164967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *MaskVec = getMaskVecValue(CGF, Ops[2], 66174967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Ops[1]->getType()->getVectorNumElements()); 66184967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 66194967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return CGF.Builder.CreateMaskedLoad(Ops[0], Align, MaskVec, Ops[1]); 66204967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar} 66214967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 66224967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainarstatic Value *EmitX86Select(CodeGenFunction &CGF, 66234967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Mask, Value *Op0, Value *Op1) { 66244967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 66254967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // If the mask is all ones just return first argument. 66264967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (const auto *C = dyn_cast<Constant>(Mask)) 66274967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (C->isAllOnesValue()) 66284967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return Op0; 66294967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 66304967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Mask = getMaskVecValue(CGF, Mask, Op0->getType()->getVectorNumElements()); 66314967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 66324967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return CGF.Builder.CreateSelect(Mask, Op0, Op1); 66334967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar} 66344967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 66354967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainarstatic Value *EmitX86MaskedCompare(CodeGenFunction &CGF, unsigned CC, 66364967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar bool Signed, SmallVectorImpl<Value *> &Ops) { 66374967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar unsigned NumElts = Ops[0]->getType()->getVectorNumElements(); 66384967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Cmp; 66394967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 66404967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (CC == 3) { 66414967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Cmp = Constant::getNullValue( 66424967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::VectorType::get(CGF.Builder.getInt1Ty(), NumElts)); 66434967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } else if (CC == 7) { 66444967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Cmp = Constant::getAllOnesValue( 66454967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::VectorType::get(CGF.Builder.getInt1Ty(), NumElts)); 66464967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } else { 66474967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar ICmpInst::Predicate Pred; 66484967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar switch (CC) { 66494967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar default: llvm_unreachable("Unknown condition code"); 66504967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case 0: Pred = ICmpInst::ICMP_EQ; break; 66514967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break; 66524967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break; 66534967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case 4: Pred = ICmpInst::ICMP_NE; break; 66544967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break; 66554967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break; 66564967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 66574967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Cmp = CGF.Builder.CreateICmp(Pred, Ops[0], Ops[1]); 66584967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 66594967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 66604967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar const auto *C = dyn_cast<Constant>(Ops.back()); 66614967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (!C || !C->isAllOnesValue()) 66624967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Cmp = CGF.Builder.CreateAnd(Cmp, getMaskVecValue(CGF, Ops.back(), NumElts)); 66634967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 66644967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (NumElts < 8) { 66654967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar uint32_t Indices[8]; 66664967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar for (unsigned i = 0; i != NumElts; ++i) 66674967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Indices[i] = i; 66684967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar for (unsigned i = NumElts; i != 8; ++i) 66694967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Indices[i] = i % NumElts + NumElts; 66704967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Cmp = CGF.Builder.CreateShuffleVector( 66714967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Cmp, llvm::Constant::getNullValue(Cmp->getType()), Indices); 66724967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 66734967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return CGF.Builder.CreateBitCast(Cmp, 66744967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar IntegerType::get(CGF.getLLVMContext(), 66754967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar std::max(NumElts, 8U))); 66764967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar} 66774967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 66781eb4433ac451dc16f4133a88af2d002ac26c58efMike StumpValue *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, 66791feedd84221e8dbcc3faf3de27cc42b559db845dChris Lattner const CallExpr *E) { 668087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar if (BuiltinID == X86::BI__builtin_ms_va_start || 668187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar BuiltinID == X86::BI__builtin_ms_va_end) 668287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).getPointer(), 668387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar BuiltinID == X86::BI__builtin_ms_va_start); 668487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar if (BuiltinID == X86::BI__builtin_ms_va_copy) { 668587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Lower this manually. We can't reliably determine whether or not any 668687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // given va_copy() is for a Win64 va_list from the calling convention 668787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // alone, because it's legal to do this from a System V ABI function. 668887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // With opaque pointer types, we won't have enough information in LLVM 668987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // IR to determine this from the argument types, either. Best to do it 669087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // now, while we have enough information. 669187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address DestAddr = EmitMSVAListRef(E->getArg(0)); 669287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address SrcAddr = EmitMSVAListRef(E->getArg(1)); 669387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 669487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *BPP = Int8PtrPtrTy; 669587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 669687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar DestAddr = Address(Builder.CreateBitCast(DestAddr.getPointer(), BPP, "cp"), 669787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar DestAddr.getAlignment()); 669887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar SrcAddr = Address(Builder.CreateBitCast(SrcAddr.getPointer(), BPP, "ap"), 669987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar SrcAddr.getAlignment()); 670087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 670187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *ArgPtr = Builder.CreateLoad(SrcAddr, "ap.val"); 670287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateStore(ArgPtr, DestAddr); 670387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 670487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 67055f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner SmallVector<Value*, 4> Ops; 67062929cfa9b7df1d5b0571b54161783e4d791a0b77Anders Carlsson 670746c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner // Find out if any arguments are required to be integer constant expressions. 670846c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner unsigned ICEArguments = 0; 670946c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner ASTContext::GetBuiltinTypeError Error; 671046c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); 671146c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner assert(Error == ASTContext::GE_None && "Should not codegen an error"); 671246c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner 671346c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) { 671446c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner // If this is a normal argument, just emit it as a scalar. 671546c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner if ((ICEArguments & (1 << i)) == 0) { 671646c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner Ops.push_back(EmitScalarExpr(E->getArg(i))); 671746c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner continue; 671846c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner } 671946c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner 672046c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner // If this is required to be a constant, constant fold it so that we know 672146c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner // that the generated intrinsic gets a ConstantInt. 672246c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner llvm::APSInt Result; 672346c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext()); 672446c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst; 6725d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); 672646c5591f0fa2e35367e44234e59bb041d15b778eChris Lattner } 67272929cfa9b7df1d5b0571b54161783e4d791a0b77Anders Carlsson 67284967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // These exist so that the builtin that takes an immediate can be bounds 67294967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // checked by clang to avoid passing bad immediates to the backend. Since 67304967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // AVX has a larger immediate than SSE we would need separate builtins to 67314967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // do the different bounds checking. Rather than create a clang specific 67324967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // SSE only builtin, this implements eight separate builtins to match gcc 67334967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // implementation. 67344967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar auto getCmpIntrinsicCall = [this, &Ops](Intrinsic::ID ID, unsigned Imm) { 67354967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Ops.push_back(llvm::ConstantInt::get(Int8Ty, Imm)); 67364967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Function *F = CGM.getIntrinsic(ID); 67374967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return Builder.CreateCall(F, Ops); 67384967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar }; 67394967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 67404967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // For the vector forms of FP comparisons, translate the builtins directly to 67414967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // IR. 67424967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // TODO: The builtins could be removed if the SSE header files used vector 67434967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // extension comparisons directly (vector ordered/unordered may need 67444967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // additional support via __builtin_isnan()). 67454967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar auto getVectorFCmpIR = [this, &Ops](CmpInst::Predicate Pred) { 67464967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]); 67474967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::VectorType *FPVecTy = cast<llvm::VectorType>(Ops[0]->getType()); 67484967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::VectorType *IntVecTy = llvm::VectorType::getInteger(FPVecTy); 67494967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Sext = Builder.CreateSExt(Cmp, IntVecTy); 67504967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return Builder.CreateBitCast(Sext, FPVecTy); 67514967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar }; 67524967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 6753564f1de67d7ba43646b8740db86d6269e3dfbe0bAnders Carlsson switch (BuiltinID) { 67546bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines default: return nullptr; 675587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case X86::BI__builtin_cpu_supports: { 675687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar const Expr *FeatureExpr = E->getArg(0)->IgnoreParenCasts(); 675787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString(); 675887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 675987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // TODO: When/if this becomes more than x86 specific then use a TargetInfo 676087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // based mapping. 676187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Processor features and mapping to processor feature value. 676287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar enum X86Features { 676387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar CMOV = 0, 676487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar MMX, 676587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar POPCNT, 676687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar SSE, 676787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar SSE2, 676887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar SSE3, 676987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar SSSE3, 677087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar SSE4_1, 677187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar SSE4_2, 677287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AVX, 677387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AVX2, 677487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar SSE4_A, 677587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar FMA4, 677687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar XOP, 677787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar FMA, 677887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AVX512F, 677987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar BMI, 678087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar BMI2, 67814967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar AES, 67824967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar PCLMUL, 67834967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar AVX512VL, 67844967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar AVX512BW, 67854967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar AVX512DQ, 67864967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar AVX512CD, 67874967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar AVX512ER, 67884967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar AVX512PF, 67894967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar AVX512VBMI, 67904967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar AVX512IFMA, 679187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar MAX 679287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar }; 679387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 679487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar X86Features Feature = StringSwitch<X86Features>(FeatureStr) 679587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar .Case("cmov", X86Features::CMOV) 679687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar .Case("mmx", X86Features::MMX) 679787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar .Case("popcnt", X86Features::POPCNT) 679887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar .Case("sse", X86Features::SSE) 679987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar .Case("sse2", X86Features::SSE2) 680087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar .Case("sse3", X86Features::SSE3) 68014967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar .Case("ssse3", X86Features::SSSE3) 680287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar .Case("sse4.1", X86Features::SSE4_1) 680387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar .Case("sse4.2", X86Features::SSE4_2) 680487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar .Case("avx", X86Features::AVX) 680587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar .Case("avx2", X86Features::AVX2) 680687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar .Case("sse4a", X86Features::SSE4_A) 680787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar .Case("fma4", X86Features::FMA4) 680887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar .Case("xop", X86Features::XOP) 680987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar .Case("fma", X86Features::FMA) 681087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar .Case("avx512f", X86Features::AVX512F) 681187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar .Case("bmi", X86Features::BMI) 681287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar .Case("bmi2", X86Features::BMI2) 68134967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar .Case("aes", X86Features::AES) 68144967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar .Case("pclmul", X86Features::PCLMUL) 68154967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar .Case("avx512vl", X86Features::AVX512VL) 68164967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar .Case("avx512bw", X86Features::AVX512BW) 68174967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar .Case("avx512dq", X86Features::AVX512DQ) 68184967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar .Case("avx512cd", X86Features::AVX512CD) 68194967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar .Case("avx512er", X86Features::AVX512ER) 68204967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar .Case("avx512pf", X86Features::AVX512PF) 68214967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar .Case("avx512vbmi", X86Features::AVX512VBMI) 68224967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar .Case("avx512ifma", X86Features::AVX512IFMA) 682387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar .Default(X86Features::MAX); 682487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar assert(Feature != X86Features::MAX && "Invalid feature!"); 682587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 682687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Matching the struct layout from the compiler-rt/libgcc structure that is 682787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // filled in: 682887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // unsigned int __cpu_vendor; 682987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // unsigned int __cpu_type; 683087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // unsigned int __cpu_subtype; 683187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // unsigned int __cpu_features[1]; 683287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *STy = llvm::StructType::get( 683387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Int32Ty, Int32Ty, Int32Ty, llvm::ArrayType::get(Int32Ty, 1), nullptr); 683487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 683587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Grab the global __cpu_model. 683687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Constant *CpuModel = CGM.CreateRuntimeVariable(STy, "__cpu_model"); 683787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 683887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Grab the first (0th) element from the field __cpu_features off of the 683987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // global in the struct STy. 684087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Idxs[] = { 684187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar ConstantInt::get(Int32Ty, 0), 684287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar ConstantInt::get(Int32Ty, 3), 684387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar ConstantInt::get(Int32Ty, 0) 684487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar }; 684587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *CpuFeatures = Builder.CreateGEP(STy, CpuModel, Idxs); 684687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Features = Builder.CreateAlignedLoad(CpuFeatures, 684787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar CharUnits::fromQuantity(4)); 684887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 684987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Check the value of the bit corresponding to the feature requested. 685087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Bitset = Builder.CreateAnd( 68514967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Features, llvm::ConstantInt::get(Int32Ty, 1ULL << Feature)); 685287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateICmpNE(Bitset, llvm::ConstantInt::get(Int32Ty, 0)); 685387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 6854651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case X86::BI_mm_prefetch: { 685587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Address = Ops[0]; 6856651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *RW = ConstantInt::get(Int32Ty, 0); 685787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Locality = Ops[1]; 6858651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *Data = ConstantInt::get(Int32Ty, 1); 6859651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Value *F = CGM.getIntrinsic(Intrinsic::prefetch); 6860b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, {Address, RW, Locality, Data}); 6861651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 686287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case X86::BI__builtin_ia32_undef128: 686387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case X86::BI__builtin_ia32_undef256: 686487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case X86::BI__builtin_ia32_undef512: 686587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return UndefValue::get(ConvertType(E->getType())); 6866aa51e513850688b7963efc62abf1eface7037602Bill Wendling case X86::BI__builtin_ia32_vec_init_v8qi: 6867aa51e513850688b7963efc62abf1eface7037602Bill Wendling case X86::BI__builtin_ia32_vec_init_v4hi: 6868aa51e513850688b7963efc62abf1eface7037602Bill Wendling case X86::BI__builtin_ia32_vec_init_v2si: 6869aa51e513850688b7963efc62abf1eface7037602Bill Wendling return Builder.CreateBitCast(BuildVector(Ops), 6870d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall llvm::Type::getX86_MMXTy(getLLVMContext())); 68711944ec188408aff1931c62c79a069e30f2549ec2Argyrios Kyrtzidis case X86::BI__builtin_ia32_vec_ext_v2si: 68721944ec188408aff1931c62c79a069e30f2549ec2Argyrios Kyrtzidis return Builder.CreateExtractElement(Ops[0], 68731944ec188408aff1931c62c79a069e30f2549ec2Argyrios Kyrtzidis llvm::ConstantInt::get(Ops[1]->getType(), 0)); 6874e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman case X86::BI__builtin_ia32_ldmxcsr: { 687587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Tmp = CreateMemTemp(E->getArg(0)->getType()); 6876e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman Builder.CreateStore(Ops[0], Tmp); 6877e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr), 687887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateBitCast(Tmp.getPointer(), Int8PtrTy)); 6879e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman } 6880e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman case X86::BI__builtin_ia32_stmxcsr: { 688187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Tmp = CreateMemTemp(E->getType()); 6882012614ecf78442368ec82ee30efb3bc047b413e6Ted Kremenek Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr), 688387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateBitCast(Tmp.getPointer(), Int8PtrTy)); 6884e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman return Builder.CreateLoad(Tmp, "stmxcsr"); 6885e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman } 688687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case X86::BI__builtin_ia32_xsave: 688787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case X86::BI__builtin_ia32_xsave64: 688887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case X86::BI__builtin_ia32_xrstor: 688987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case X86::BI__builtin_ia32_xrstor64: 689087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case X86::BI__builtin_ia32_xsaveopt: 689187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case X86::BI__builtin_ia32_xsaveopt64: 689287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case X86::BI__builtin_ia32_xrstors: 689387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case X86::BI__builtin_ia32_xrstors64: 689487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case X86::BI__builtin_ia32_xsavec: 689587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case X86::BI__builtin_ia32_xsavec64: 689687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case X86::BI__builtin_ia32_xsaves: 689787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case X86::BI__builtin_ia32_xsaves64: { 689887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Intrinsic::ID ID; 689987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar#define INTRINSIC_X86_XSAVE_ID(NAME) \ 690087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case X86::BI__builtin_ia32_##NAME: \ 690187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar ID = Intrinsic::x86_##NAME; \ 690287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar break 690387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar switch (BuiltinID) { 690487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar default: llvm_unreachable("Unsupported intrinsic!"); 690587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar INTRINSIC_X86_XSAVE_ID(xsave); 690687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar INTRINSIC_X86_XSAVE_ID(xsave64); 690787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar INTRINSIC_X86_XSAVE_ID(xrstor); 690887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar INTRINSIC_X86_XSAVE_ID(xrstor64); 690987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar INTRINSIC_X86_XSAVE_ID(xsaveopt); 691087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar INTRINSIC_X86_XSAVE_ID(xsaveopt64); 691187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar INTRINSIC_X86_XSAVE_ID(xrstors); 691287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar INTRINSIC_X86_XSAVE_ID(xrstors64); 691387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar INTRINSIC_X86_XSAVE_ID(xsavec); 691487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar INTRINSIC_X86_XSAVE_ID(xsavec64); 691587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar INTRINSIC_X86_XSAVE_ID(xsaves); 691687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar INTRINSIC_X86_XSAVE_ID(xsaves64); 691787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 691887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar#undef INTRINSIC_X86_XSAVE_ID 691987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Mhi = Builder.CreateTrunc( 692087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, 32)), Int32Ty); 692187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Mlo = Builder.CreateTrunc(Ops[1], Int32Ty); 692287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[1] = Mhi; 692387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops.push_back(Mlo); 692487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateCall(CGM.getIntrinsic(ID), Ops); 692587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 69264967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_storedqudi128_mask: 69274967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_storedqusi128_mask: 69284967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_storedquhi128_mask: 69294967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_storedquqi128_mask: 69304967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_storeupd128_mask: 69314967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_storeups128_mask: 69324967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_storedqudi256_mask: 69334967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_storedqusi256_mask: 69344967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_storedquhi256_mask: 69354967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_storedquqi256_mask: 69364967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_storeupd256_mask: 69374967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_storeups256_mask: 69384967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_storedqudi512_mask: 69394967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_storedqusi512_mask: 69404967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_storedquhi512_mask: 69414967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_storedquqi512_mask: 69424967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_storeupd512_mask: 69434967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_storeups512_mask: 69444967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return EmitX86MaskedStore(*this, Ops, 1); 69454967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 69464967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_movdqa32store128_mask: 69474967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_movdqa64store128_mask: 69484967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_storeaps128_mask: 69494967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_storeapd128_mask: 69504967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_movdqa32store256_mask: 69514967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_movdqa64store256_mask: 69524967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_storeaps256_mask: 69534967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_storeapd256_mask: 69544967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_movdqa32store512_mask: 69554967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_movdqa64store512_mask: 69564967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_storeaps512_mask: 69574967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_storeapd512_mask: { 69584967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar unsigned Align = 69594967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar getContext().getTypeAlignInChars(E->getArg(1)->getType()).getQuantity(); 69604967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return EmitX86MaskedStore(*this, Ops, Align); 69614967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 69624967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_loadups128_mask: 69634967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_loadups256_mask: 69644967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_loadups512_mask: 69654967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_loadupd128_mask: 69664967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_loadupd256_mask: 69674967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_loadupd512_mask: 69684967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_loaddquqi128_mask: 69694967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_loaddquqi256_mask: 69704967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_loaddquqi512_mask: 69714967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_loaddquhi128_mask: 69724967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_loaddquhi256_mask: 69734967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_loaddquhi512_mask: 69744967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_loaddqusi128_mask: 69754967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_loaddqusi256_mask: 69764967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_loaddqusi512_mask: 69774967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_loaddqudi128_mask: 69784967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_loaddqudi256_mask: 69794967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_loaddqudi512_mask: 69804967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return EmitX86MaskedLoad(*this, Ops, 1); 69814967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 69824967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_loadaps128_mask: 69834967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_loadaps256_mask: 69844967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_loadaps512_mask: 69854967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_loadapd128_mask: 69864967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_loadapd256_mask: 69874967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_loadapd512_mask: 69884967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_movdqa32load128_mask: 69894967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_movdqa32load256_mask: 69904967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_movdqa32load512_mask: 69914967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_movdqa64load128_mask: 69924967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_movdqa64load256_mask: 69934967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_movdqa64load512_mask: { 69944967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar unsigned Align = 69954967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar getContext().getTypeAlignInChars(E->getArg(1)->getType()).getQuantity(); 69964967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return EmitX86MaskedLoad(*this, Ops, Align); 69974967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 6998e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman case X86::BI__builtin_ia32_storehps: 6999e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman case X86::BI__builtin_ia32_storelps: { 700077b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner llvm::Type *PtrTy = llvm::PointerType::getUnqual(Int64Ty); 700177b89b87c3b9220fea1bc80f6d6598d2003cc8a8Chris Lattner llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2); 70021eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 7003e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman // cast val v2i64 7004e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman Ops[1] = Builder.CreateBitCast(Ops[1], VecTy, "cast"); 70051eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 7006e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman // extract (0, 1) 7007e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman unsigned Index = BuiltinID == X86::BI__builtin_ia32_storelps ? 0 : 1; 7008c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Idx = llvm::ConstantInt::get(SizeTy, Index); 7009e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman Ops[1] = Builder.CreateExtractElement(Ops[1], Idx, "extract"); 7010e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman 7011e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman // cast pointer to i64 & store 7012e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman Ops[0] = Builder.CreateBitCast(Ops[0], PtrTy); 701387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 7014e7722103abc4583366c914374d6aa8560e145fa1Nate Begeman } 70150e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case X86::BI__builtin_ia32_palignr128: 70164967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_palignr256: 70174967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_palignr128_mask: 70184967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_palignr256_mask: 70194967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_palignr512_mask: { 70200e2c34f92f00628d48968dfea096d36381f494cbStephen Hines unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); 70210e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 70224967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar unsigned NumElts = Ops[0]->getType()->getVectorNumElements(); 70230e2c34f92f00628d48968dfea096d36381f494cbStephen Hines assert(NumElts % 16 == 0); 70240e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 70250e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // If palignr is shifting the pair of vectors more than the size of two 70260e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // lanes, emit zero. 70274967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (ShiftVal >= 32) 70280e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return llvm::Constant::getNullValue(ConvertType(E->getType())); 70290e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 70300e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // If palignr is shifting the pair of input vectors more than one lane, 70310e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // but less than two lanes, convert to shifting in zeroes. 70324967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (ShiftVal > 16) { 70334967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar ShiftVal -= 16; 703487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[1] = Ops[0]; 70350e2c34f92f00628d48968dfea096d36381f494cbStephen Hines Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType()); 703628cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling } 7037258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 70384967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar uint32_t Indices[64]; 70390e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // 256-bit palignr operates on 128-bit lanes so we need to handle that 70404967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar for (unsigned l = 0; l != NumElts; l += 16) { 70414967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar for (unsigned i = 0; i != 16; ++i) { 70420e2c34f92f00628d48968dfea096d36381f494cbStephen Hines unsigned Idx = ShiftVal + i; 70434967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (Idx >= 16) 70444967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Idx += NumElts - 16; // End of lane, switch operand. 704587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Indices[l + i] = Idx + l; 70460e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 704728cab383fd9e7647d2186340eca769303cc4fbdbBill Wendling } 7048258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 70494967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Align = Builder.CreateShuffleVector(Ops[1], Ops[0], 70504967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar makeArrayRef(Indices, NumElts), 70514967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar "palignr"); 7052258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 70534967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // If this isn't a masked builtin, just return the align operation. 70544967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (Ops.size() == 3) 70554967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return Align; 7056258f930227c1a102c9c22eee88df65f748863425Jim Grosbach 70574967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return EmitX86Select(*this, Ops[4], Align, Ops[3]); 70580e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 70590e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 7060440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman case X86::BI__builtin_ia32_movnti: 7061440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman case X86::BI__builtin_ia32_movnti64: { 70620e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::MDNode *Node = llvm::MDNode::get( 70630e2c34f92f00628d48968dfea096d36381f494cbStephen Hines getLLVMContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1))); 7064b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling 7065b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling // Convert the type of the pointer to a pointer to the stored type. 7066b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling Value *BC = Builder.CreateBitCast(Ops[0], 7067b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling llvm::PointerType::getUnqual(Ops[1]->getType()), 7068b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling "cast"); 706987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar StoreInst *SI = Builder.CreateDefaultAlignedStore(Ops[1], BC); 7070b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling SI->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node); 7071440a5f49133307745de7cc92a44d53088cf47c26Eli Friedman 70724967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // No alignment for scalar intrinsic store. 70734967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SI->setAlignment(1); 70744967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return SI; 70754967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 70764967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_movntsd: 70774967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_movntss: { 70784967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::MDNode *Node = llvm::MDNode::get( 70794967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar getLLVMContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1))); 70804967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 70814967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Extract the 0'th element of the source vector. 70824967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Scl = Builder.CreateExtractElement(Ops[1], (uint64_t)0, "extract"); 70834967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 70844967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Convert the type of the pointer to a pointer to the stored type. 70854967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *BC = Builder.CreateBitCast(Ops[0], 70864967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::PointerType::getUnqual(Scl->getType()), 70874967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar "cast"); 70884967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 70894967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Unaligned nontemporal store of the scalar value. 70904967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar StoreInst *SI = Builder.CreateDefaultAlignedStore(Scl, BC); 70914967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SI->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node); 70924967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SI->setAlignment(1); 7093b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling return SI; 7094b51bddab211ba7daa8832c017d82281e0d8348d1Bill Wendling } 70954967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 70964967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_selectb_128: 70974967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_selectb_256: 70984967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_selectb_512: 70994967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_selectw_128: 71004967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_selectw_256: 71014967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_selectw_512: 71024967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_selectd_128: 71034967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_selectd_256: 71044967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_selectd_512: 71054967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_selectq_128: 71064967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_selectq_256: 71074967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_selectq_512: 71084967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_selectps_128: 71094967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_selectps_256: 71104967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_selectps_512: 71114967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_selectpd_128: 71124967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_selectpd_256: 71134967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_selectpd_512: 71144967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return EmitX86Select(*this, Ops[0], Ops[1], Ops[2]); 71154967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pcmpeqb128_mask: 71164967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pcmpeqb256_mask: 71174967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pcmpeqb512_mask: 71184967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pcmpeqw128_mask: 71194967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pcmpeqw256_mask: 71204967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pcmpeqw512_mask: 71214967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pcmpeqd128_mask: 71224967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pcmpeqd256_mask: 71234967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pcmpeqd512_mask: 71244967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pcmpeqq128_mask: 71254967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pcmpeqq256_mask: 71264967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pcmpeqq512_mask: 71274967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return EmitX86MaskedCompare(*this, 0, false, Ops); 71284967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pcmpgtb128_mask: 71294967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pcmpgtb256_mask: 71304967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pcmpgtb512_mask: 71314967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pcmpgtw128_mask: 71324967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pcmpgtw256_mask: 71334967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pcmpgtw512_mask: 71344967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pcmpgtd128_mask: 71354967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pcmpgtd256_mask: 71364967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pcmpgtd512_mask: 71374967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pcmpgtq128_mask: 71384967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pcmpgtq256_mask: 71394967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pcmpgtq512_mask: 71404967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return EmitX86MaskedCompare(*this, 6, true, Ops); 71414967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmpb128_mask: 71424967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmpb256_mask: 71434967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmpb512_mask: 71444967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmpw128_mask: 71454967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmpw256_mask: 71464967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmpw512_mask: 71474967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmpd128_mask: 71484967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmpd256_mask: 71494967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmpd512_mask: 71504967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmpq128_mask: 71514967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmpq256_mask: 71524967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmpq512_mask: { 71534967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7; 71544967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return EmitX86MaskedCompare(*this, CC, true, Ops); 71554967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 71564967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_ucmpb128_mask: 71574967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_ucmpb256_mask: 71584967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_ucmpb512_mask: 71594967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_ucmpw128_mask: 71604967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_ucmpw256_mask: 71614967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_ucmpw512_mask: 71624967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_ucmpd128_mask: 71634967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_ucmpd256_mask: 71644967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_ucmpd512_mask: 71654967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_ucmpq128_mask: 71664967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_ucmpq256_mask: 71674967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_ucmpq512_mask: { 71684967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7; 71694967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return EmitX86MaskedCompare(*this, CC, false, Ops); 71704967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 71714967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 71724967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_vplzcntd_128_mask: 71734967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_vplzcntd_256_mask: 71744967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_vplzcntd_512_mask: 71754967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_vplzcntq_128_mask: 71764967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_vplzcntq_256_mask: 71774967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_vplzcntq_512_mask: { 71784967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType()); 71794967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return EmitX86Select(*this, Ops[2], 71804967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Builder.CreateCall(F, {Ops[0],Builder.getInt1(false)}), 71814967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Ops[1]); 71824967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 71834967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 71844967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // TODO: Handle 64/512-bit vector widths of min/max. 71854967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pmaxsb128: 71864967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pmaxsw128: 71874967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pmaxsd128: 71884967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pmaxsb256: 71894967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pmaxsw256: 71904967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pmaxsd256: { 71914967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_SGT, Ops[0], Ops[1]); 71924967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return Builder.CreateSelect(Cmp, Ops[0], Ops[1]); 71934967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 71944967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pmaxub128: 71954967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pmaxuw128: 71964967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pmaxud128: 71974967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pmaxub256: 71984967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pmaxuw256: 71994967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pmaxud256: { 72004967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_UGT, Ops[0], Ops[1]); 72014967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return Builder.CreateSelect(Cmp, Ops[0], Ops[1]); 72024967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 72034967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pminsb128: 72044967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pminsw128: 72054967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pminsd128: 72064967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pminsb256: 72074967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pminsw256: 72084967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pminsd256: { 72094967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_SLT, Ops[0], Ops[1]); 72104967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return Builder.CreateSelect(Cmp, Ops[0], Ops[1]); 72114967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 72124967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pminub128: 72134967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pminuw128: 72144967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pminud128: 72154967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pminub256: 72164967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pminuw256: 72174967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_pminud256: { 72184967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_ULT, Ops[0], Ops[1]); 72194967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return Builder.CreateSelect(Cmp, Ops[0], Ops[1]); 72204967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 72214967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 72228b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer // 3DNow! 72238b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer case X86::BI__builtin_ia32_pswapdsf: 72248b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer case X86::BI__builtin_ia32_pswapdsi: { 7225345032a7211a6f983d59c30c0b3fa2b96819532aChandler Carruth llvm::Type *MMXTy = llvm::Type::getX86_MMXTy(getLLVMContext()); 7226345032a7211a6f983d59c30c0b3fa2b96819532aChandler Carruth Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy, "cast"); 72270e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_3dnowa_pswapd); 72280e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return Builder.CreateCall(F, Ops, "pswapd"); 72298b36a9ee7fe7204b30a85b95b11850aeb4b63ee3Michael J. Spencer } 72309a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer case X86::BI__builtin_ia32_rdrand16_step: 72319a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer case X86::BI__builtin_ia32_rdrand32_step: 72321bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao case X86::BI__builtin_ia32_rdrand64_step: 72331bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao case X86::BI__builtin_ia32_rdseed16_step: 72341bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao case X86::BI__builtin_ia32_rdseed32_step: 72351bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao case X86::BI__builtin_ia32_rdseed64_step: { 72369a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer Intrinsic::ID ID; 72379a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer switch (BuiltinID) { 72389a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer default: llvm_unreachable("Unsupported intrinsic!"); 72399a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer case X86::BI__builtin_ia32_rdrand16_step: 72409a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer ID = Intrinsic::x86_rdrand_16; 72419a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer break; 72429a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer case X86::BI__builtin_ia32_rdrand32_step: 72439a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer ID = Intrinsic::x86_rdrand_32; 72449a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer break; 72459a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer case X86::BI__builtin_ia32_rdrand64_step: 72469a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer ID = Intrinsic::x86_rdrand_64; 72479a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer break; 72481bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao case X86::BI__builtin_ia32_rdseed16_step: 72491bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao ID = Intrinsic::x86_rdseed_16; 72501bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao break; 72511bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao case X86::BI__builtin_ia32_rdseed32_step: 72521bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao ID = Intrinsic::x86_rdseed_32; 72531bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao break; 72541bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao case X86::BI__builtin_ia32_rdseed64_step: 72551bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao ID = Intrinsic::x86_rdseed_64; 72561bfc28c48c1b86a05d2e07b403107ef3da5a0f8eMichael Liao break; 72579a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer } 72589a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer 725987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID)); 726087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 0), 726187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ops[0]); 72629a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer return Builder.CreateExtractValue(Call, 1); 72639a50249cd35f3d9d4d2b194a3edd6815ccf746d7Benjamin Kramer } 72644967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 72654967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // SSE packed comparison intrinsics 72660e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case X86::BI__builtin_ia32_cmpeqps: 72674967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmpeqpd: 72684967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return getVectorFCmpIR(CmpInst::FCMP_OEQ); 72690e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case X86::BI__builtin_ia32_cmpltps: 72704967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmpltpd: 72714967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return getVectorFCmpIR(CmpInst::FCMP_OLT); 72720e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case X86::BI__builtin_ia32_cmpleps: 72734967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmplepd: 72744967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return getVectorFCmpIR(CmpInst::FCMP_OLE); 72750e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case X86::BI__builtin_ia32_cmpunordps: 72764967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmpunordpd: 72774967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return getVectorFCmpIR(CmpInst::FCMP_UNO); 72780e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case X86::BI__builtin_ia32_cmpneqps: 72794967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmpneqpd: 72804967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return getVectorFCmpIR(CmpInst::FCMP_UNE); 72810e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case X86::BI__builtin_ia32_cmpnltps: 72824967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmpnltpd: 72834967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return getVectorFCmpIR(CmpInst::FCMP_UGE); 72840e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case X86::BI__builtin_ia32_cmpnleps: 72854967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmpnlepd: 72864967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return getVectorFCmpIR(CmpInst::FCMP_UGT); 72870e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case X86::BI__builtin_ia32_cmpordps: 72884967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmpordpd: 72894967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return getVectorFCmpIR(CmpInst::FCMP_ORD); 72904967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmpps: 72914967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmpps256: 72924967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmppd: 72934967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmppd256: { 72944967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); 72954967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // If this one of the SSE immediates, we can use native IR. 72964967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (CC < 8) { 72974967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar FCmpInst::Predicate Pred; 72984967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar switch (CC) { 72994967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case 0: Pred = FCmpInst::FCMP_OEQ; break; 73004967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case 1: Pred = FCmpInst::FCMP_OLT; break; 73014967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case 2: Pred = FCmpInst::FCMP_OLE; break; 73024967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case 3: Pred = FCmpInst::FCMP_UNO; break; 73034967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case 4: Pred = FCmpInst::FCMP_UNE; break; 73044967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case 5: Pred = FCmpInst::FCMP_UGE; break; 73054967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case 6: Pred = FCmpInst::FCMP_UGT; break; 73064967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case 7: Pred = FCmpInst::FCMP_ORD; break; 73074967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 73084967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return getVectorFCmpIR(Pred); 73094967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 73104967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 73114967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // We can't handle 8-31 immediates with native IR, use the intrinsic. 73124967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Intrinsic::ID ID; 73134967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar switch (BuiltinID) { 73144967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar default: llvm_unreachable("Unsupported intrinsic!"); 73154967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmpps: 73164967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar ID = Intrinsic::x86_sse_cmp_ps; 73174967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar break; 73184967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmpps256: 73194967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar ID = Intrinsic::x86_avx_cmp_ps_256; 73204967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar break; 73214967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmppd: 73224967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar ID = Intrinsic::x86_sse2_cmp_pd; 73234967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar break; 73244967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case X86::BI__builtin_ia32_cmppd256: 73254967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar ID = Intrinsic::x86_avx_cmp_pd_256; 73264967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar break; 73274967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 73284967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 73294967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return Builder.CreateCall(CGM.getIntrinsic(ID), Ops); 73304967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 73314967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 73324967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // SSE scalar comparison intrinsics 73330e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case X86::BI__builtin_ia32_cmpeqss: 73344967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 0); 73350e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case X86::BI__builtin_ia32_cmpltss: 73364967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 1); 73370e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case X86::BI__builtin_ia32_cmpless: 73384967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 2); 73390e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case X86::BI__builtin_ia32_cmpunordss: 73404967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 3); 73410e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case X86::BI__builtin_ia32_cmpneqss: 73424967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 4); 73430e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case X86::BI__builtin_ia32_cmpnltss: 73444967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 5); 73450e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case X86::BI__builtin_ia32_cmpnless: 73464967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 6); 73470e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case X86::BI__builtin_ia32_cmpordss: 73484967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 7); 73490e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case X86::BI__builtin_ia32_cmpeqsd: 73504967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 0); 73510e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case X86::BI__builtin_ia32_cmpltsd: 73524967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 1); 73530e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case X86::BI__builtin_ia32_cmplesd: 73544967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 2); 73550e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case X86::BI__builtin_ia32_cmpunordsd: 73564967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 3); 73570e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case X86::BI__builtin_ia32_cmpneqsd: 73584967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 4); 73590e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case X86::BI__builtin_ia32_cmpnltsd: 73604967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 5); 73610e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case X86::BI__builtin_ia32_cmpnlesd: 73624967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 6); 73630e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case X86::BI__builtin_ia32_cmpordsd: 73644967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 7); 7365564f1de67d7ba43646b8740db86d6269e3dfbe0bAnders Carlsson } 7366564f1de67d7ba43646b8740db86d6269e3dfbe0bAnders Carlsson} 7367564f1de67d7ba43646b8740db86d6269e3dfbe0bAnders Carlsson 73689631939f82c0eaa6fb3936a0ce58a41adfbc9011Tony Linthicum 73691eb4433ac451dc16f4133a88af2d002ac26c58efMike StumpValue *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, 73701feedd84221e8dbcc3faf3de27cc42b559db845dChris Lattner const CallExpr *E) { 73715f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner SmallVector<Value*, 4> Ops; 7372dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner 7373dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) 7374dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner Ops.push_back(EmitScalarExpr(E->getArg(i))); 7375dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner 7376dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner Intrinsic::ID ID = Intrinsic::not_intrinsic; 7377dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner 7378dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner switch (BuiltinID) { 73796bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines default: return nullptr; 7380dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner 738187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // __builtin_ppc_get_timebase is GCC 4.8+'s PowerPC-specific name for what we 738287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // call __builtin_readcyclecounter. 738387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_ppc_get_timebase: 738487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::readcyclecounter)); 738587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 73864d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov // vec_ld, vec_lvsl, vec_lvsr 73874d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvx: 73884d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvxl: 73894d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvebx: 73904d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvehx: 73914d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvewx: 73924d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvsl: 73934d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvsr: 7394176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case PPC::BI__builtin_vsx_lxvd2x: 7395176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case PPC::BI__builtin_vsx_lxvw4x: 73964d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov { 7397d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy); 73984d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov 7399578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer Ops[0] = Builder.CreateGEP(Ops[1], Ops[0]); 74004d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov Ops.pop_back(); 74014d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov 74024d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov switch (BuiltinID) { 7403b219cfc4d75f0a03630b7c4509ef791b7e97b2c8David Blaikie default: llvm_unreachable("Unsupported ld/lvsl/lvsr intrinsic!"); 74044d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvx: 74054d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov ID = Intrinsic::ppc_altivec_lvx; 74064d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov break; 74074d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvxl: 74084d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov ID = Intrinsic::ppc_altivec_lvxl; 74094d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov break; 74104d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvebx: 74114d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov ID = Intrinsic::ppc_altivec_lvebx; 74124d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov break; 74134d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvehx: 74144d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov ID = Intrinsic::ppc_altivec_lvehx; 74154d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov break; 74164d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvewx: 74174d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov ID = Intrinsic::ppc_altivec_lvewx; 74184d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov break; 74194d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvsl: 74204d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov ID = Intrinsic::ppc_altivec_lvsl; 74214d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov break; 74224d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov case PPC::BI__builtin_altivec_lvsr: 74234d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov ID = Intrinsic::ppc_altivec_lvsr; 74244d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov break; 7425176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case PPC::BI__builtin_vsx_lxvd2x: 7426176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines ID = Intrinsic::ppc_vsx_lxvd2x; 7427176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines break; 7428176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case PPC::BI__builtin_vsx_lxvw4x: 7429176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines ID = Intrinsic::ppc_vsx_lxvw4x; 7430176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines break; 74314d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov } 74324d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov llvm::Function *F = CGM.getIntrinsic(ID); 74334c7d9f1507d0f102bd4133bba63348636facd469Jay Foad return Builder.CreateCall(F, Ops, ""); 74344d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov } 74354d3a7b0a0608febe3cdac68f6121546672ca875eAnton Korobeynikov 7436dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner // vec_st 7437dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvx: 7438dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvxl: 7439dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvebx: 7440dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvehx: 7441dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvewx: 7442176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case PPC::BI__builtin_vsx_stxvd2x: 7443176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case PPC::BI__builtin_vsx_stxvw4x: 7444dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner { 7445d16c2cf1cafa413709aa487cbbd5dc392f1ba1ffJohn McCall Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy); 7446578faa837b552403e2002b97fdfbfde14f2448e5Benjamin Kramer Ops[1] = Builder.CreateGEP(Ops[2], Ops[1]); 7447dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner Ops.pop_back(); 7448dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner 7449dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner switch (BuiltinID) { 7450b219cfc4d75f0a03630b7c4509ef791b7e97b2c8David Blaikie default: llvm_unreachable("Unsupported st intrinsic!"); 7451dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvx: 7452dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner ID = Intrinsic::ppc_altivec_stvx; 7453dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner break; 7454dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvxl: 7455dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner ID = Intrinsic::ppc_altivec_stvxl; 7456dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner break; 7457dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvebx: 7458dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner ID = Intrinsic::ppc_altivec_stvebx; 7459dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner break; 7460dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvehx: 7461dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner ID = Intrinsic::ppc_altivec_stvehx; 7462dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner break; 7463dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner case PPC::BI__builtin_altivec_stvewx: 7464dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner ID = Intrinsic::ppc_altivec_stvewx; 7465dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner break; 7466176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case PPC::BI__builtin_vsx_stxvd2x: 7467176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines ID = Intrinsic::ppc_vsx_stxvd2x; 7468176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines break; 7469176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines case PPC::BI__builtin_vsx_stxvw4x: 7470176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines ID = Intrinsic::ppc_vsx_stxvw4x; 7471176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines break; 7472dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner } 7473dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner llvm::Function *F = CGM.getIntrinsic(ID); 74744c7d9f1507d0f102bd4133bba63348636facd469Jay Foad return Builder.CreateCall(F, Ops, ""); 7475dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner } 747687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Square root 747787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvsqrtsp: 747887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvsqrtdp: { 747987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *ResultType = ConvertType(E->getType()); 748087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *X = EmitScalarExpr(E->getArg(0)); 748187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar ID = Intrinsic::sqrt; 748287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Function *F = CGM.getIntrinsic(ID, ResultType); 748387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateCall(F, X); 748487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 748587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Count leading zeros 748687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_altivec_vclzb: 748787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_altivec_vclzh: 748887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_altivec_vclzw: 748987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_altivec_vclzd: { 749087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *ResultType = ConvertType(E->getType()); 749187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *X = EmitScalarExpr(E->getArg(0)); 749287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false); 749387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType); 749487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateCall(F, {X, Undef}); 749587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 749687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Copy sign 749787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvcpsgnsp: 749887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvcpsgndp: { 749987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *ResultType = ConvertType(E->getType()); 750087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *X = EmitScalarExpr(E->getArg(0)); 750187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Y = EmitScalarExpr(E->getArg(1)); 750287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar ID = Intrinsic::copysign; 750387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Function *F = CGM.getIntrinsic(ID, ResultType); 750487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateCall(F, {X, Y}); 750587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 750687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Rounding/truncation 750787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvrspip: 750887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvrdpip: 750987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvrdpim: 751087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvrspim: 751187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvrdpi: 751287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvrspi: 751387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvrdpic: 751487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvrspic: 751587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvrdpiz: 751687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvrspiz: { 751787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *ResultType = ConvertType(E->getType()); 751887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *X = EmitScalarExpr(E->getArg(0)); 751987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar if (BuiltinID == PPC::BI__builtin_vsx_xvrdpim || 752087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar BuiltinID == PPC::BI__builtin_vsx_xvrspim) 752187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar ID = Intrinsic::floor; 752287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpi || 752387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar BuiltinID == PPC::BI__builtin_vsx_xvrspi) 752487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar ID = Intrinsic::round; 752587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpic || 752687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar BuiltinID == PPC::BI__builtin_vsx_xvrspic) 752787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar ID = Intrinsic::nearbyint; 752887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpip || 752987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar BuiltinID == PPC::BI__builtin_vsx_xvrspip) 753087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar ID = Intrinsic::ceil; 753187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpiz || 753287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar BuiltinID == PPC::BI__builtin_vsx_xvrspiz) 753387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar ID = Intrinsic::trunc; 753487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Function *F = CGM.getIntrinsic(ID, ResultType); 753587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateCall(F, X); 753687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 75374967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 75384967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Absolute value 75394967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvabsdp: 75404967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvabssp: { 75414967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Type *ResultType = ConvertType(E->getType()); 75424967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *X = EmitScalarExpr(E->getArg(0)); 75434967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType); 75444967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return Builder.CreateCall(F, X); 75454967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 75464967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 754787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // FMA variations 754887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvmaddadp: 754987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvmaddasp: 755087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvnmaddadp: 755187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvnmaddasp: 755287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvmsubadp: 755387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvmsubasp: 755487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvnmsubadp: 755587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvnmsubasp: { 755687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *ResultType = ConvertType(E->getType()); 755787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *X = EmitScalarExpr(E->getArg(0)); 755887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Y = EmitScalarExpr(E->getArg(1)); 755987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Z = EmitScalarExpr(E->getArg(2)); 756087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType); 756187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType); 756287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar switch (BuiltinID) { 756387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvmaddadp: 756487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvmaddasp: 756587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateCall(F, {X, Y, Z}); 756687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvnmaddadp: 756787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvnmaddasp: 756887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateFSub(Zero, 756987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateCall(F, {X, Y, Z}), "sub"); 757087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvmsubadp: 757187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvmsubasp: 757287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateCall(F, 757387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar {X, Y, Builder.CreateFSub(Zero, Z, "sub")}); 757487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvnmsubadp: 757587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case PPC::BI__builtin_vsx_xvnmsubasp: 757687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *FsubRes = 757787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateCall(F, {X, Y, Builder.CreateFSub(Zero, Z, "sub")}); 757887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateFSub(Zero, FsubRes, "sub"); 757987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 758087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm_unreachable("Unknown FMA operation"); 758187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return nullptr; // Suppress no-return warning 758287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 7583dd17394d225b06376e9ae1d23f36cec463fdef01Chris Lattner } 75841eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump} 7585c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 758687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga NainarValue *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, 758787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar const CallExpr *E) { 7588c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines switch (BuiltinID) { 75894967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_div_scale: 75904967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_div_scalef: { 7591c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines // Translate from the intrinsics's struct return to the builtin's out 7592c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines // argument. 7593c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 759487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address FlagOutPtr = EmitPointerWithAlignment(E->getArg(3)); 7595c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 7596c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *X = EmitScalarExpr(E->getArg(0)); 7597c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Y = EmitScalarExpr(E->getArg(1)); 7598c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Z = EmitScalarExpr(E->getArg(2)); 7599c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 76004967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Value *Callee = CGM.getIntrinsic(Intrinsic::amdgcn_div_scale, 7601c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines X->getType()); 7602c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 7603b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Value *Tmp = Builder.CreateCall(Callee, {X, Y, Z}); 7604c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 7605c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Result = Builder.CreateExtractValue(Tmp, 0); 7606c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Flag = Builder.CreateExtractValue(Tmp, 1); 7607c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 7608c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Type *RealFlagType 760987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar = FlagOutPtr.getPointer()->getType()->getPointerElementType(); 7610c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 7611c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *FlagExt = Builder.CreateZExt(Flag, RealFlagType); 761287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateStore(FlagExt, FlagOutPtr); 7613c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines return Result; 7614176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 76154967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_div_fmas: 76164967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_div_fmasf: { 7617176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::Value *Src0 = EmitScalarExpr(E->getArg(0)); 7618176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::Value *Src1 = EmitScalarExpr(E->getArg(1)); 7619176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::Value *Src2 = EmitScalarExpr(E->getArg(2)); 7620176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::Value *Src3 = EmitScalarExpr(E->getArg(3)); 7621176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 76224967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::Value *F = CGM.getIntrinsic(Intrinsic::amdgcn_div_fmas, 7623176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Src0->getType()); 7624176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::Value *Src3ToBool = Builder.CreateIsNotNull(Src3); 7625b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool}); 7626176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 76274967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_div_fixup: 76284967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_div_fixupf: 76294967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return emitTernaryBuiltin(*this, E, Intrinsic::amdgcn_div_fixup); 76304967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_trig_preop: 76314967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_trig_preopf: 76324967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_trig_preop); 76334967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_rcp: 76344967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_rcpf: 76354967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rcp); 76364967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_rsq: 76374967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_rsqf: 76384967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rsq); 76394967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_rsq_clamp: 76404967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_rsq_clampf: 76414967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rsq_clamp); 76424967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_sinf: 76434967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_sin); 76444967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_cosf: 76454967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_cos); 76464967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_log_clampf: 76474967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_log_clamp); 76484967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_ldexp: 76494967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_ldexpf: 76504967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_ldexp); 76514967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_frexp_mant: 76524967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_frexp_mantf: { 76534967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_frexp_mant); 76544967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 76554967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_frexp_exp: 76564967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_frexp_expf: { 76574967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_frexp_exp); 76584967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 76594967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_fract: 76604967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_fractf: 76614967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_fract); 76624967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_class: 76634967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_classf: 76644967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_class); 76654967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 76664967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_read_exec: { 76674967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar CallInst *CI = cast<CallInst>( 76684967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar EmitSpecialRegisterBuiltin(*this, E, Int64Ty, Int64Ty, true, "exec")); 76694967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar CI->setConvergent(); 76704967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return CI; 76714967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 76724967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Legacy amdgpu prefix 767387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgpu_rsq: 76744967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgpu_rsqf: { 76754967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (getTarget().getTriple().getArch() == Triple::amdgcn) 76764967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rsq); 76774967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return emitUnaryBuiltin(*this, E, Intrinsic::r600_rsq); 76784967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 767987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgpu_ldexp: 76804967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgpu_ldexpf: { 76814967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (getTarget().getTriple().getArch() == Triple::amdgcn) 76824967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_ldexp); 7683176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return emitFPIntBuiltin(*this, E, Intrinsic::AMDGPU_ldexp); 76844967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 76854967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 76864967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // amdgcn workitem 76874967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_workitem_id_x: 76884967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return emitRangedBuiltin(*this, Intrinsic::amdgcn_workitem_id_x, 0, 1024); 76894967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_workitem_id_y: 76904967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return emitRangedBuiltin(*this, Intrinsic::amdgcn_workitem_id_y, 0, 1024); 76914967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_amdgcn_workitem_id_z: 76924967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return emitRangedBuiltin(*this, Intrinsic::amdgcn_workitem_id_z, 0, 1024); 76934967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 76944967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // r600 workitem 76954967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_r600_read_tidig_x: 76964967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return emitRangedBuiltin(*this, Intrinsic::r600_read_tidig_x, 0, 1024); 76974967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_r600_read_tidig_y: 76984967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return emitRangedBuiltin(*this, Intrinsic::r600_read_tidig_y, 0, 1024); 76994967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case AMDGPU::BI__builtin_r600_read_tidig_z: 77004967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return emitRangedBuiltin(*this, Intrinsic::r600_read_tidig_z, 0, 1024); 77014967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar default: 7702c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines return nullptr; 7703c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines } 7704c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines} 770558878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar 7706b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar/// Handle a SystemZ function in which the final argument is a pointer 7707b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar/// to an int that receives the post-instruction CC value. At the LLVM level 7708b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar/// this is represented as a function that returns a {result, cc} pair. 7709b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainarstatic Value *EmitSystemZIntrinsicWithCC(CodeGenFunction &CGF, 7710b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar unsigned IntrinsicID, 7711b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar const CallExpr *E) { 7712b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar unsigned NumArgs = E->getNumArgs() - 1; 7713b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar SmallVector<Value *, 8> Args(NumArgs); 7714b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar for (unsigned I = 0; I < NumArgs; ++I) 7715b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Args[I] = CGF.EmitScalarExpr(E->getArg(I)); 771687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address CCPtr = CGF.EmitPointerWithAlignment(E->getArg(NumArgs)); 7717b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *F = CGF.CGM.getIntrinsic(IntrinsicID); 7718b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *Call = CGF.Builder.CreateCall(F, Args); 7719b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *CC = CGF.Builder.CreateExtractValue(Call, 1); 7720b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar CGF.Builder.CreateStore(CC, CCPtr); 7721b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return CGF.Builder.CreateExtractValue(Call, 0); 7722b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar} 7723b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 772458878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga NainarValue *CodeGenFunction::EmitSystemZBuiltinExpr(unsigned BuiltinID, 772558878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar const CallExpr *E) { 772658878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar switch (BuiltinID) { 772758878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar case SystemZ::BI__builtin_tbegin: { 772858878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar Value *TDB = EmitScalarExpr(E->getArg(0)); 772958878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff0c); 773058878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar Value *F = CGM.getIntrinsic(Intrinsic::s390_tbegin); 7731b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, {TDB, Control}); 773258878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar } 773358878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar case SystemZ::BI__builtin_tbegin_nofloat: { 773458878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar Value *TDB = EmitScalarExpr(E->getArg(0)); 773558878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff0c); 773658878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar Value *F = CGM.getIntrinsic(Intrinsic::s390_tbegin_nofloat); 7737b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, {TDB, Control}); 773858878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar } 773958878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar case SystemZ::BI__builtin_tbeginc: { 774058878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar Value *TDB = llvm::ConstantPointerNull::get(Int8PtrTy); 774158878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff08); 774258878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar Value *F = CGM.getIntrinsic(Intrinsic::s390_tbeginc); 7743b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, {TDB, Control}); 774458878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar } 774558878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar case SystemZ::BI__builtin_tabort: { 774658878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar Value *Data = EmitScalarExpr(E->getArg(0)); 774758878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar Value *F = CGM.getIntrinsic(Intrinsic::s390_tabort); 774858878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar return Builder.CreateCall(F, Builder.CreateSExt(Data, Int64Ty, "tabort")); 774958878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar } 775058878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar case SystemZ::BI__builtin_non_tx_store: { 775158878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar Value *Address = EmitScalarExpr(E->getArg(0)); 775258878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar Value *Data = EmitScalarExpr(E->getArg(1)); 775358878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar Value *F = CGM.getIntrinsic(Intrinsic::s390_ntstg); 7754b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, {Data, Address}); 7755b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 7756b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 7757b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // Vector builtins. Note that most vector builtins are mapped automatically 7758b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // to target-specific LLVM intrinsics. The ones handled specially here can 7759b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // be represented via standard LLVM IR, which is preferable to enable common 7760b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // LLVM optimizations. 7761b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 7762b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case SystemZ::BI__builtin_s390_vpopctb: 7763b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case SystemZ::BI__builtin_s390_vpopcth: 7764b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case SystemZ::BI__builtin_s390_vpopctf: 7765b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case SystemZ::BI__builtin_s390_vpopctg: { 7766b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Type *ResultType = ConvertType(E->getType()); 7767b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *X = EmitScalarExpr(E->getArg(0)); 7768b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType); 7769b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, X); 7770b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 7771b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 7772b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case SystemZ::BI__builtin_s390_vclzb: 7773b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case SystemZ::BI__builtin_s390_vclzh: 7774b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case SystemZ::BI__builtin_s390_vclzf: 7775b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case SystemZ::BI__builtin_s390_vclzg: { 7776b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Type *ResultType = ConvertType(E->getType()); 7777b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *X = EmitScalarExpr(E->getArg(0)); 7778b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false); 7779b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType); 7780b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, {X, Undef}); 7781b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 7782b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 7783b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case SystemZ::BI__builtin_s390_vctzb: 7784b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case SystemZ::BI__builtin_s390_vctzh: 7785b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case SystemZ::BI__builtin_s390_vctzf: 7786b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case SystemZ::BI__builtin_s390_vctzg: { 7787b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Type *ResultType = ConvertType(E->getType()); 7788b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *X = EmitScalarExpr(E->getArg(0)); 7789b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false); 7790b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Function *F = CGM.getIntrinsic(Intrinsic::cttz, ResultType); 7791b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, {X, Undef}); 7792b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 7793b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 7794b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case SystemZ::BI__builtin_s390_vfsqdb: { 7795b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Type *ResultType = ConvertType(E->getType()); 7796b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *X = EmitScalarExpr(E->getArg(0)); 7797b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Function *F = CGM.getIntrinsic(Intrinsic::sqrt, ResultType); 7798b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, X); 7799b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 7800b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case SystemZ::BI__builtin_s390_vfmadb: { 7801b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Type *ResultType = ConvertType(E->getType()); 7802b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *X = EmitScalarExpr(E->getArg(0)); 7803b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *Y = EmitScalarExpr(E->getArg(1)); 7804b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *Z = EmitScalarExpr(E->getArg(2)); 7805b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType); 7806b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, {X, Y, Z}); 7807b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 7808b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case SystemZ::BI__builtin_s390_vfmsdb: { 7809b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Type *ResultType = ConvertType(E->getType()); 7810b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *X = EmitScalarExpr(E->getArg(0)); 7811b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *Y = EmitScalarExpr(E->getArg(1)); 7812b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *Z = EmitScalarExpr(E->getArg(2)); 7813b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType); 7814b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType); 7815b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, {X, Y, Builder.CreateFSub(Zero, Z, "sub")}); 7816b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 7817b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case SystemZ::BI__builtin_s390_vflpdb: { 7818b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Type *ResultType = ConvertType(E->getType()); 7819b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *X = EmitScalarExpr(E->getArg(0)); 7820b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType); 7821b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, X); 782258878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar } 7823b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case SystemZ::BI__builtin_s390_vflndb: { 7824b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Type *ResultType = ConvertType(E->getType()); 7825b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *X = EmitScalarExpr(E->getArg(0)); 7826b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType); 7827b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType); 7828b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateFSub(Zero, Builder.CreateCall(F, X), "sub"); 7829b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 7830b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case SystemZ::BI__builtin_s390_vfidb: { 7831b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Type *ResultType = ConvertType(E->getType()); 7832b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *X = EmitScalarExpr(E->getArg(0)); 7833b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // Constant-fold the M4 and M5 mask arguments. 7834b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::APSInt M4, M5; 7835b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar bool IsConstM4 = E->getArg(1)->isIntegerConstantExpr(M4, getContext()); 7836b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar bool IsConstM5 = E->getArg(2)->isIntegerConstantExpr(M5, getContext()); 7837b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar assert(IsConstM4 && IsConstM5 && "Constant arg isn't actually constant?"); 7838b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar (void)IsConstM4; (void)IsConstM5; 7839b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // Check whether this instance of vfidb can be represented via a LLVM 7840b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // standard intrinsic. We only support some combinations of M4 and M5. 7841b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Intrinsic::ID ID = Intrinsic::not_intrinsic; 7842b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar switch (M4.getZExtValue()) { 7843b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar default: break; 7844b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case 0: // IEEE-inexact exception allowed 7845b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar switch (M5.getZExtValue()) { 7846b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar default: break; 7847b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case 0: ID = Intrinsic::rint; break; 7848b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 7849b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar break; 7850b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case 4: // IEEE-inexact exception suppressed 7851b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar switch (M5.getZExtValue()) { 7852b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar default: break; 7853b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case 0: ID = Intrinsic::nearbyint; break; 7854b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case 1: ID = Intrinsic::round; break; 7855b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case 5: ID = Intrinsic::trunc; break; 7856b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case 6: ID = Intrinsic::ceil; break; 7857b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case 7: ID = Intrinsic::floor; break; 7858b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 7859b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar break; 7860b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 7861b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar if (ID != Intrinsic::not_intrinsic) { 7862b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Function *F = CGM.getIntrinsic(ID, ResultType); 7863b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, X); 7864b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 7865b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Function *F = CGM.getIntrinsic(Intrinsic::s390_vfidb); 7866b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *M4Value = llvm::ConstantInt::get(getLLVMContext(), M4); 7867b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Value *M5Value = llvm::ConstantInt::get(getLLVMContext(), M5); 7868b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return Builder.CreateCall(F, {X, M4Value, M5Value}); 7869b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 7870b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 7871b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // Vector intrisincs that output the post-instruction CC value. 7872b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 7873b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar#define INTRINSIC_WITH_CC(NAME) \ 7874b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar case SystemZ::BI__builtin_##NAME: \ 7875b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return EmitSystemZIntrinsicWithCC(*this, Intrinsic::NAME, E) 7876b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 7877b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vpkshs); 7878b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vpksfs); 7879b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vpksgs); 7880b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 7881b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vpklshs); 7882b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vpklsfs); 7883b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vpklsgs); 7884b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 7885b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vceqbs); 7886b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vceqhs); 7887b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vceqfs); 7888b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vceqgs); 7889b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 7890b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vchbs); 7891b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vchhs); 7892b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vchfs); 7893b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vchgs); 7894b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 7895b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vchlbs); 7896b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vchlhs); 7897b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vchlfs); 7898b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vchlgs); 7899b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 7900b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vfaebs); 7901b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vfaehs); 7902b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vfaefs); 7903b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 7904b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vfaezbs); 7905b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vfaezhs); 7906b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vfaezfs); 7907b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 7908b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vfeebs); 7909b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vfeehs); 7910b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vfeefs); 7911b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 7912b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vfeezbs); 7913b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vfeezhs); 7914b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vfeezfs); 7915b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 7916b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vfenebs); 7917b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vfenehs); 7918b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vfenefs); 7919b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 7920b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vfenezbs); 7921b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vfenezhs); 7922b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vfenezfs); 7923b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 7924b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vistrbs); 7925b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vistrhs); 7926b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vistrfs); 7927b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 7928b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vstrcbs); 7929b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vstrchs); 7930b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vstrcfs); 7931b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 7932b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vstrczbs); 7933b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vstrczhs); 7934b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vstrczfs); 7935b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 7936b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vfcedbs); 7937b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vfchdbs); 7938b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vfchedbs); 7939b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 7940b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar INTRINSIC_WITH_CC(s390_vftcidb); 7941b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 7942b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar#undef INTRINSIC_WITH_CC 794358878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar 794458878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar default: 794558878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar return nullptr; 794658878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar } 794758878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar} 794887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 794987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga NainarValue *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, 795087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar const CallExpr *E) { 79514967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar auto MakeLdg = [&](unsigned IntrinsicID) { 79524967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Ptr = EmitScalarExpr(E->getArg(0)); 79534967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar AlignmentSource AlignSource; 79544967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar clang::CharUnits Align = 79554967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar getNaturalPointeeTypeAlignment(E->getArg(0)->getType(), &AlignSource); 79564967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return Builder.CreateCall( 79574967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar CGM.getIntrinsic(IntrinsicID, {Ptr->getType()->getPointerElementType(), 79584967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Ptr->getType()}), 79594967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar {Ptr, ConstantInt::get(Builder.getInt32Ty(), Align.getQuantity())}); 79604967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar }; 79614967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 796287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar switch (BuiltinID) { 796387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_add_gen_i: 796487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_add_gen_l: 796587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_add_gen_ll: 796687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Add, E); 796787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 796887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_sub_gen_i: 796987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_sub_gen_l: 797087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_sub_gen_ll: 797187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Sub, E); 797287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 797387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_and_gen_i: 797487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_and_gen_l: 797587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_and_gen_ll: 797687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::And, E); 797787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 797887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_or_gen_i: 797987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_or_gen_l: 798087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_or_gen_ll: 798187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Or, E); 798287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 798387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_xor_gen_i: 798487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_xor_gen_l: 798587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_xor_gen_ll: 798687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Xor, E); 798787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 798887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_xchg_gen_i: 798987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_xchg_gen_l: 799087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_xchg_gen_ll: 799187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Xchg, E); 799287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 799387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_max_gen_i: 799487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_max_gen_l: 799587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_max_gen_ll: 799687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Max, E); 799787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 799887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_max_gen_ui: 799987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_max_gen_ul: 800087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_max_gen_ull: 800187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::UMax, E); 800287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 800387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_min_gen_i: 800487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_min_gen_l: 800587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_min_gen_ll: 800687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Min, E); 800787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 800887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_min_gen_ui: 800987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_min_gen_ul: 801087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_min_gen_ull: 801187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::UMin, E); 801287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 801387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_cas_gen_i: 801487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_cas_gen_l: 801587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_cas_gen_ll: 801687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // __nvvm_atom_cas_gen_* should return the old value rather than the 801787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // success flag. 801887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return MakeAtomicCmpXchgValue(*this, E, /*ReturnBool=*/false); 801987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 802087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_add_gen_f: { 802187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Ptr = EmitScalarExpr(E->getArg(0)); 802287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Val = EmitScalarExpr(E->getArg(1)); 802387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // atomicrmw only deals with integer arguments so we need to use 802487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // LLVM's nvvm_atomic_load_add_f32 intrinsic for that. 802587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *FnALAF32 = 802687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_add_f32, Ptr->getType()); 802787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateCall(FnALAF32, {Ptr, Val}); 802887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 802987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 80304967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_inc_gen_ui: { 80314967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Ptr = EmitScalarExpr(E->getArg(0)); 80324967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Val = EmitScalarExpr(E->getArg(1)); 80334967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *FnALI32 = 80344967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_inc_32, Ptr->getType()); 80354967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return Builder.CreateCall(FnALI32, {Ptr, Val}); 80364967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 80374967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 80384967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_atom_dec_gen_ui: { 80394967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Ptr = EmitScalarExpr(E->getArg(0)); 80404967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Val = EmitScalarExpr(E->getArg(1)); 80414967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *FnALD32 = 80424967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_dec_32, Ptr->getType()); 80434967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return Builder.CreateCall(FnALD32, {Ptr, Val}); 80444967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 80454967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar 80464967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_c: 80474967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_c2: 80484967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_c4: 80494967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_s: 80504967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_s2: 80514967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_s4: 80524967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_i: 80534967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_i2: 80544967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_i4: 80554967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_l: 80564967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_ll: 80574967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_ll2: 80584967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_uc: 80594967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_uc2: 80604967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_uc4: 80614967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_us: 80624967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_us2: 80634967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_us4: 80644967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_ui: 80654967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_ui2: 80664967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_ui4: 80674967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_ul: 80684967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_ull: 80694967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_ull2: 80704967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // PTX Interoperability section 2.2: "For a vector with an even number of 80714967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // elements, its alignment is set to number of elements times the alignment 80724967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // of its member: n*alignof(t)." 80734967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return MakeLdg(Intrinsic::nvvm_ldg_global_i); 80744967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_f: 80754967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_f2: 80764967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_f4: 80774967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_d: 80784967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case NVPTX::BI__nvvm_ldg_d2: 80794967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return MakeLdg(Intrinsic::nvvm_ldg_global_f); 808087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar default: 808187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return nullptr; 808287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 808387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 808487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 808587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga NainarValue *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, 808687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar const CallExpr *E) { 808787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar switch (BuiltinID) { 80884967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case WebAssembly::BI__builtin_wasm_current_memory: { 808987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *ResultType = ConvertType(E->getType()); 80904967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_current_memory, ResultType); 809187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateCall(Callee); 809287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 809387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case WebAssembly::BI__builtin_wasm_grow_memory: { 809487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *X = EmitScalarExpr(E->getArg(0)); 809587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_grow_memory, X->getType()); 809687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Builder.CreateCall(Callee, X); 809787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 809887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 809987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar default: 810087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return nullptr; 810187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 810287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 8103