16994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===// 26994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth// 36994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth// The LLVM Compiler Infrastructure 46994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth// 54ee451de366474b9c228b4e5fa573795a715216dChris Lattner// This file is distributed under the University of Illinois Open Source 64ee451de366474b9c228b4e5fa573795a715216dChris Lattner// License. See LICENSE.TXT for details. 76994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth// 86994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth//===----------------------------------------------------------------------===// 96994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth// 106994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth// This file implements the auto-upgrade helper functions 116994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth// 126994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth//===----------------------------------------------------------------------===// 136994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth 146994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth#include "llvm/AutoUpgrade.h" 15d04764a8acabe0b3f0452b3099bc5964ba783884Anders Carlsson#include "llvm/Constants.h" 166994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth#include "llvm/Function.h" 1706cb8ed00696eb14d1b831921452e50ec0568ea2Chandler Carruth#include "llvm/IRBuilder.h" 18c82a61c6c611857e47c3d6c6dbd069c27cd6d0f2Bill Wendling#include "llvm/Instruction.h" 1906cb8ed00696eb14d1b831921452e50ec0568ea2Chandler Carruth#include "llvm/IntrinsicInst.h" 20f81b57694b3c34a79b1464ffd21e6768c8d22662Owen Anderson#include "llvm/LLVMContext.h" 216994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth#include "llvm/Module.h" 22c82a61c6c611857e47c3d6c6dbd069c27cd6d0f2Bill Wendling#include "llvm/Support/CFG.h" 2306cb8ed00696eb14d1b831921452e50ec0568ea2Chandler Carruth#include "llvm/Support/CallSite.h" 24c25e7581b9b8088910da31702d4ca21c4734c6d7Torok Edwin#include "llvm/Support/ErrorHandling.h" 25ae9f3a3b7c915f725aef5a7250e88eaeddda03c6Anton Korobeynikov#include <cstring> 266994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruthusing namespace llvm; 276994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth 283c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem// Upgrade the declarations of the SSE4.1 functions whose arguments have 293c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem// changed their type from v4f32 to v2i64. 303c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotemstatic bool UpgradeSSE41Function(Function* F, Intrinsic::ID IID, 313c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem Function *&NewFn) { 323c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem // Check whether this is an old version of the function, which received 333c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem // v4f32 arguments. 343c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem Type *Arg0Type = F->getFunctionType()->getParamType(0); 353c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem if (Arg0Type != VectorType::get(Type::getFloatTy(F->getContext()), 4)) 363c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem return false; 373c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem 383c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem // Yes, it's old, replace it with new version. 393c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem F->setName(F->getName() + ".old"); 403c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem NewFn = Intrinsic::getDeclaration(F->getParent(), IID); 413c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem return true; 423c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem} 436994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth 44f9b83fcf95e8a280d7b117af8858596fe5b10d94Evan Chengstatic bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { 456994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth assert(F && "Illegal to upgrade a non-existent Function."); 466994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth 476994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth // Quickly eliminate it, if it's not a candidate. 48747fddd48450d75794c4f58c020f54785b4bc66fChris Lattner StringRef Name = F->getName(); 49747fddd48450d75794c4f58c020f54785b4bc66fChris Lattner if (Name.size() <= 8 || !Name.startswith("llvm.")) 50f9b83fcf95e8a280d7b117af8858596fe5b10d94Evan Cheng return false; 51747fddd48450d75794c4f58c020f54785b4bc66fChris Lattner Name = Name.substr(5); // Strip off "llvm." 52b5dd9de7240f3018dcd8ce84c158547a5e0f1131Chris Lattner 53747fddd48450d75794c4f58c020f54785b4bc66fChris Lattner switch (Name[0]) { 546994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth default: break; 5506a6a300c5f7100e4665667c689369e078d2ad59Joel Jones case 'a': { 5606a6a300c5f7100e4665667c689369e078d2ad59Joel Jones if (Name.startswith("arm.neon.vclz")) { 5706a6a300c5f7100e4665667c689369e078d2ad59Joel Jones Type* args[2] = { 5806a6a300c5f7100e4665667c689369e078d2ad59Joel Jones F->arg_begin()->getType(), 5906a6a300c5f7100e4665667c689369e078d2ad59Joel Jones Type::getInt1Ty(F->getContext()) 6006a6a300c5f7100e4665667c689369e078d2ad59Joel Jones }; 6106a6a300c5f7100e4665667c689369e078d2ad59Joel Jones // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to 6206a6a300c5f7100e4665667c689369e078d2ad59Joel Jones // the end of the name. Change name from llvm.arm.neon.vclz.* to 6306a6a300c5f7100e4665667c689369e078d2ad59Joel Jones // llvm.ctlz.* 6406a6a300c5f7100e4665667c689369e078d2ad59Joel Jones FunctionType* fType = FunctionType::get(F->getReturnType(), args, false); 6506a6a300c5f7100e4665667c689369e078d2ad59Joel Jones NewFn = Function::Create(fType, F->getLinkage(), 6606a6a300c5f7100e4665667c689369e078d2ad59Joel Jones "llvm.ctlz." + Name.substr(14), F->getParent()); 6706a6a300c5f7100e4665667c689369e078d2ad59Joel Jones return true; 6806a6a300c5f7100e4665667c689369e078d2ad59Joel Jones } 697c82e6a32a84e238c9e4e57dd43eaba540a79ce1Joel Jones if (Name.startswith("arm.neon.vcnt")) { 707c82e6a32a84e238c9e4e57dd43eaba540a79ce1Joel Jones NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop, 717c82e6a32a84e238c9e4e57dd43eaba540a79ce1Joel Jones F->arg_begin()->getType()); 727c82e6a32a84e238c9e4e57dd43eaba540a79ce1Joel Jones return true; 737c82e6a32a84e238c9e4e57dd43eaba540a79ce1Joel Jones } 7406a6a300c5f7100e4665667c689369e078d2ad59Joel Jones break; 7506a6a300c5f7100e4665667c689369e078d2ad59Joel Jones } 76ccbf1e36d34c2d66600748bdd8767dc122629a58Chandler Carruth case 'c': { 77ccbf1e36d34c2d66600748bdd8767dc122629a58Chandler Carruth if (Name.startswith("ctlz.") && F->arg_size() == 1) { 78ccbf1e36d34c2d66600748bdd8767dc122629a58Chandler Carruth F->setName(Name + ".old"); 79a56f5581ecb6ba741645a74bfa399f4bdf5b2f71Chandler Carruth NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz, 80a56f5581ecb6ba741645a74bfa399f4bdf5b2f71Chandler Carruth F->arg_begin()->getType()); 81ccbf1e36d34c2d66600748bdd8767dc122629a58Chandler Carruth return true; 82ccbf1e36d34c2d66600748bdd8767dc122629a58Chandler Carruth } 83ccbf1e36d34c2d66600748bdd8767dc122629a58Chandler Carruth if (Name.startswith("cttz.") && F->arg_size() == 1) { 84ccbf1e36d34c2d66600748bdd8767dc122629a58Chandler Carruth F->setName(Name + ".old"); 85a56f5581ecb6ba741645a74bfa399f4bdf5b2f71Chandler Carruth NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz, 86a56f5581ecb6ba741645a74bfa399f4bdf5b2f71Chandler Carruth F->arg_begin()->getType()); 87ccbf1e36d34c2d66600748bdd8767dc122629a58Chandler Carruth return true; 88ccbf1e36d34c2d66600748bdd8767dc122629a58Chandler Carruth } 89ccbf1e36d34c2d66600748bdd8767dc122629a58Chandler Carruth break; 90ccbf1e36d34c2d66600748bdd8767dc122629a58Chandler Carruth } 91e058d276559bcecb11662c2548b249876e9430b0Craig Topper case 'x': { 92e058d276559bcecb11662c2548b249876e9430b0Craig Topper if (Name.startswith("x86.sse2.pcmpeq.") || 93e058d276559bcecb11662c2548b249876e9430b0Craig Topper Name.startswith("x86.sse2.pcmpgt.") || 94e058d276559bcecb11662c2548b249876e9430b0Craig Topper Name.startswith("x86.avx2.pcmpeq.") || 95a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper Name.startswith("x86.avx2.pcmpgt.") || 96189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper Name.startswith("x86.avx.vpermil.") || 97189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper Name == "x86.avx.movnt.dq.256" || 98189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper Name == "x86.avx.movnt.pd.256" || 99c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper Name == "x86.avx.movnt.ps.256" || 100c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper (Name.startswith("x86.xop.vpcom") && F->arg_size() == 2)) { 101e058d276559bcecb11662c2548b249876e9430b0Craig Topper NewFn = 0; 102e058d276559bcecb11662c2548b249876e9430b0Craig Topper return true; 103e058d276559bcecb11662c2548b249876e9430b0Craig Topper } 1043c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem // SSE4.1 ptest functions may have an old signature. 1053c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem if (Name.startswith("x86.sse41.ptest")) { 1063c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem if (Name == "x86.sse41.ptestc") 1073c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestc, NewFn); 1083c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem if (Name == "x86.sse41.ptestz") 1093c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestz, NewFn); 1103c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem if (Name == "x86.sse41.ptestnzc") 1113c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestnzc, NewFn); 1123c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem } 113cc95b57d42a4af1cbb0a0e4a4efc2133116dd21cCraig Topper // frcz.ss/sd may need to have an argument dropped 114cc95b57d42a4af1cbb0a0e4a4efc2133116dd21cCraig Topper if (Name.startswith("x86.xop.vfrcz.ss") && F->arg_size() == 2) { 115cc95b57d42a4af1cbb0a0e4a4efc2133116dd21cCraig Topper F->setName(Name + ".old"); 116cc95b57d42a4af1cbb0a0e4a4efc2133116dd21cCraig Topper NewFn = Intrinsic::getDeclaration(F->getParent(), 117cc95b57d42a4af1cbb0a0e4a4efc2133116dd21cCraig Topper Intrinsic::x86_xop_vfrcz_ss); 118cc95b57d42a4af1cbb0a0e4a4efc2133116dd21cCraig Topper return true; 119cc95b57d42a4af1cbb0a0e4a4efc2133116dd21cCraig Topper } 120cc95b57d42a4af1cbb0a0e4a4efc2133116dd21cCraig Topper if (Name.startswith("x86.xop.vfrcz.sd") && F->arg_size() == 2) { 121cc95b57d42a4af1cbb0a0e4a4efc2133116dd21cCraig Topper F->setName(Name + ".old"); 122cc95b57d42a4af1cbb0a0e4a4efc2133116dd21cCraig Topper NewFn = Intrinsic::getDeclaration(F->getParent(), 123cc95b57d42a4af1cbb0a0e4a4efc2133116dd21cCraig Topper Intrinsic::x86_xop_vfrcz_sd); 124cc95b57d42a4af1cbb0a0e4a4efc2133116dd21cCraig Topper return true; 125cc95b57d42a4af1cbb0a0e4a4efc2133116dd21cCraig Topper } 1267678de45b27b6a07ea28113c415ccfcb68871950Craig Topper // Fix the FMA4 intrinsics to remove the 4 1277678de45b27b6a07ea28113c415ccfcb68871950Craig Topper if (Name.startswith("x86.fma4.")) { 128bb6e61cf6b11dcda6ca37bc895ac2cd1a7a96486Craig Topper F->setName("llvm.x86.fma" + Name.substr(8)); 129bb6e61cf6b11dcda6ca37bc895ac2cd1a7a96486Craig Topper NewFn = F; 130bb6e61cf6b11dcda6ca37bc895ac2cd1a7a96486Craig Topper return true; 1317678de45b27b6a07ea28113c415ccfcb68871950Craig Topper } 132e058d276559bcecb11662c2548b249876e9430b0Craig Topper break; 133e058d276559bcecb11662c2548b249876e9430b0Craig Topper } 134747fddd48450d75794c4f58c020f54785b4bc66fChris Lattner } 1356994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth 1363c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem // This may not belong here. This function is effectively being overloaded 1373c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem // to both detect an intrinsic which needs upgrading, and to provide the 1383c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem // upgraded form of the intrinsic. We should perhaps have two separate 1396994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth // functions for this. 140f9b83fcf95e8a280d7b117af8858596fe5b10d94Evan Cheng return false; 1416994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth} 1426994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth 143f9b83fcf95e8a280d7b117af8858596fe5b10d94Evan Chengbool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) { 144f9b83fcf95e8a280d7b117af8858596fe5b10d94Evan Cheng NewFn = 0; 145f9b83fcf95e8a280d7b117af8858596fe5b10d94Evan Cheng bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn); 146a3355ffb3d30d19d226bbb75707991c60f236e37Duncan Sands 147a3355ffb3d30d19d226bbb75707991c60f236e37Duncan Sands // Upgrade intrinsic attributes. This does not change the function. 148f9b83fcf95e8a280d7b117af8858596fe5b10d94Evan Cheng if (NewFn) 149f9b83fcf95e8a280d7b117af8858596fe5b10d94Evan Cheng F = NewFn; 15049de98214b82fefeb8f16efbf8cdd8813a85469bDale Johannesen if (unsigned id = F->getIntrinsicID()) 1510598866c052147c31b808391f58434ce3dbfb838Devang Patel F->setAttributes(Intrinsic::getAttributes((Intrinsic::ID)id)); 152a3355ffb3d30d19d226bbb75707991c60f236e37Duncan Sands return Upgraded; 153a3355ffb3d30d19d226bbb75707991c60f236e37Duncan Sands} 154a3355ffb3d30d19d226bbb75707991c60f236e37Duncan Sands 155de49f360ec4780acb382a3ae923d1716fdb0dc03Bill Wendlingbool llvm::UpgradeGlobalVariable(GlobalVariable *GV) { 156b85e4eba85a38698f3b3332f82554bf8442547e2Chris Lattner // Nothing to do yet. 157de49f360ec4780acb382a3ae923d1716fdb0dc03Bill Wendling return false; 158de49f360ec4780acb382a3ae923d1716fdb0dc03Bill Wendling} 159de49f360ec4780acb382a3ae923d1716fdb0dc03Bill Wendling 1603c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem// UpgradeIntrinsicCall - Upgrade a call to an old intrinsic to be a call the 1613c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem// upgraded intrinsic. All argument and return casting must be provided in 1626994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth// order to seamlessly integrate with existing context. 1636994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruthvoid llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { 164e058d276559bcecb11662c2548b249876e9430b0Craig Topper Function *F = CI->getCalledFunction(); 165bf47c7627812714b9427c7c847d6880194a76af6Nick Lewycky LLVMContext &C = CI->getContext(); 166ccbf1e36d34c2d66600748bdd8767dc122629a58Chandler Carruth IRBuilder<> Builder(C); 167ccbf1e36d34c2d66600748bdd8767dc122629a58Chandler Carruth Builder.SetInsertPoint(CI->getParent(), CI); 168ccbf1e36d34c2d66600748bdd8767dc122629a58Chandler Carruth 169e058d276559bcecb11662c2548b249876e9430b0Craig Topper assert(F && "Intrinsic call is not direct?"); 170e058d276559bcecb11662c2548b249876e9430b0Craig Topper 171e058d276559bcecb11662c2548b249876e9430b0Craig Topper if (!NewFn) { 172e058d276559bcecb11662c2548b249876e9430b0Craig Topper // Get the Function's name. 173e058d276559bcecb11662c2548b249876e9430b0Craig Topper StringRef Name = F->getName(); 174e058d276559bcecb11662c2548b249876e9430b0Craig Topper 175e058d276559bcecb11662c2548b249876e9430b0Craig Topper Value *Rep; 176e058d276559bcecb11662c2548b249876e9430b0Craig Topper // Upgrade packed integer vector compares intrinsics to compare instructions 177e058d276559bcecb11662c2548b249876e9430b0Craig Topper if (Name.startswith("llvm.x86.sse2.pcmpeq.") || 178e058d276559bcecb11662c2548b249876e9430b0Craig Topper Name.startswith("llvm.x86.avx2.pcmpeq.")) { 179e058d276559bcecb11662c2548b249876e9430b0Craig Topper Rep = Builder.CreateICmpEQ(CI->getArgOperand(0), CI->getArgOperand(1), 180e058d276559bcecb11662c2548b249876e9430b0Craig Topper "pcmpeq"); 181e058d276559bcecb11662c2548b249876e9430b0Craig Topper // need to sign extend since icmp returns vector of i1 182e058d276559bcecb11662c2548b249876e9430b0Craig Topper Rep = Builder.CreateSExt(Rep, CI->getType(), ""); 183e058d276559bcecb11662c2548b249876e9430b0Craig Topper } else if (Name.startswith("llvm.x86.sse2.pcmpgt.") || 184e058d276559bcecb11662c2548b249876e9430b0Craig Topper Name.startswith("llvm.x86.avx2.pcmpgt.")) { 185e058d276559bcecb11662c2548b249876e9430b0Craig Topper Rep = Builder.CreateICmpSGT(CI->getArgOperand(0), CI->getArgOperand(1), 186e058d276559bcecb11662c2548b249876e9430b0Craig Topper "pcmpgt"); 187e058d276559bcecb11662c2548b249876e9430b0Craig Topper // need to sign extend since icmp returns vector of i1 188e058d276559bcecb11662c2548b249876e9430b0Craig Topper Rep = Builder.CreateSExt(Rep, CI->getType(), ""); 189189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper } else if (Name == "llvm.x86.avx.movnt.dq.256" || 190189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper Name == "llvm.x86.avx.movnt.ps.256" || 191189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper Name == "llvm.x86.avx.movnt.pd.256") { 192189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper IRBuilder<> Builder(C); 193189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper Builder.SetInsertPoint(CI->getParent(), CI); 194189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper 195189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper Module *M = F->getParent(); 196189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper SmallVector<Value *, 1> Elts; 197189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper Elts.push_back(ConstantInt::get(Type::getInt32Ty(C), 1)); 198189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper MDNode *Node = MDNode::get(C, Elts); 199189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper 200189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper Value *Arg0 = CI->getArgOperand(0); 201189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper Value *Arg1 = CI->getArgOperand(1); 202189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper 203189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper // Convert the type of the pointer to a pointer to the stored type. 204189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper Value *BC = Builder.CreateBitCast(Arg0, 205189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper PointerType::getUnqual(Arg1->getType()), 206189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper "cast"); 207189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper StoreInst *SI = Builder.CreateStore(Arg1, BC); 208189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper SI->setMetadata(M->getMDKindID("nontemporal"), Node); 209189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper SI->setAlignment(16); 210189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper 211189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper // Remove intrinsic. 212189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper CI->eraseFromParent(); 213189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper return; 214c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper } else if (Name.startswith("llvm.x86.xop.vpcom")) { 215c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper Intrinsic::ID intID; 216c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper if (Name.endswith("ub")) 217c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper intID = Intrinsic::x86_xop_vpcomub; 218c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper else if (Name.endswith("uw")) 219c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper intID = Intrinsic::x86_xop_vpcomuw; 220c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper else if (Name.endswith("ud")) 221c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper intID = Intrinsic::x86_xop_vpcomud; 222c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper else if (Name.endswith("uq")) 223c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper intID = Intrinsic::x86_xop_vpcomuq; 224c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper else if (Name.endswith("b")) 225c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper intID = Intrinsic::x86_xop_vpcomb; 226c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper else if (Name.endswith("w")) 227c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper intID = Intrinsic::x86_xop_vpcomw; 228c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper else if (Name.endswith("d")) 229c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper intID = Intrinsic::x86_xop_vpcomd; 230c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper else if (Name.endswith("q")) 231c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper intID = Intrinsic::x86_xop_vpcomq; 232c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper else 233c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper llvm_unreachable("Unknown suffix"); 234c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper 235c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper Name = Name.substr(18); // strip off "llvm.x86.xop.vpcom" 236c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper unsigned Imm; 237c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper if (Name.startswith("lt")) 238c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper Imm = 0; 239c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper else if (Name.startswith("le")) 240c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper Imm = 1; 241c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper else if (Name.startswith("gt")) 242c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper Imm = 2; 243c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper else if (Name.startswith("ge")) 244c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper Imm = 3; 245c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper else if (Name.startswith("eq")) 246c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper Imm = 4; 247c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper else if (Name.startswith("ne")) 248c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper Imm = 5; 249c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper else if (Name.startswith("true")) 250c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper Imm = 6; 251c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper else if (Name.startswith("false")) 252c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper Imm = 7; 253c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper else 254c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper llvm_unreachable("Unknown condition"); 255c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper 256c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper Function *VPCOM = Intrinsic::getDeclaration(F->getParent(), intID); 257c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper Rep = Builder.CreateCall3(VPCOM, CI->getArgOperand(0), 258c29106b36f97d0f2dc806d1e8bf8d44fc466b9d3Craig Topper CI->getArgOperand(1), Builder.getInt8(Imm)); 259e058d276559bcecb11662c2548b249876e9430b0Craig Topper } else { 260a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper bool PD128 = false, PD256 = false, PS128 = false, PS256 = false; 261189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper if (Name == "llvm.x86.avx.vpermil.pd.256") 262a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper PD256 = true; 263189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper else if (Name == "llvm.x86.avx.vpermil.pd") 264a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper PD128 = true; 265189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper else if (Name == "llvm.x86.avx.vpermil.ps.256") 266a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper PS256 = true; 267189bce48c7121b0d21f23aa3dec91fcf66e27925Craig Topper else if (Name == "llvm.x86.avx.vpermil.ps") 268a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper PS128 = true; 269a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper 270a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper if (PD256 || PD128 || PS256 || PS128) { 271a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper Value *Op0 = CI->getArgOperand(0); 272a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 273a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper SmallVector<Constant*, 8> Idxs; 274a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper 275a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper if (PD128) 276a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper for (unsigned i = 0; i != 2; ++i) 277a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper Idxs.push_back(Builder.getInt32((Imm >> i) & 0x1)); 278a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper else if (PD256) 279a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper for (unsigned l = 0; l != 4; l+=2) 280a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper for (unsigned i = 0; i != 2; ++i) 281a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper Idxs.push_back(Builder.getInt32(((Imm >> (l+i)) & 0x1) + l)); 282a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper else if (PS128) 283a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper for (unsigned i = 0; i != 4; ++i) 284a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper Idxs.push_back(Builder.getInt32((Imm >> (2 * i)) & 0x3)); 285a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper else if (PS256) 286a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper for (unsigned l = 0; l != 8; l+=4) 287a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper for (unsigned i = 0; i != 4; ++i) 288a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper Idxs.push_back(Builder.getInt32(((Imm >> (2 * i)) & 0x3) + l)); 289a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper else 290a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper llvm_unreachable("Unexpected function"); 291a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper 292a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper Rep = Builder.CreateShuffleVector(Op0, Op0, ConstantVector::get(Idxs)); 293a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper } else { 294a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper llvm_unreachable("Unknown function for CallInst upgrade."); 295a963c81819bed30d54174cdaf5a7e58696838d5aCraig Topper } 296e058d276559bcecb11662c2548b249876e9430b0Craig Topper } 297e058d276559bcecb11662c2548b249876e9430b0Craig Topper 298e058d276559bcecb11662c2548b249876e9430b0Craig Topper CI->replaceAllUsesWith(Rep); 299e058d276559bcecb11662c2548b249876e9430b0Craig Topper CI->eraseFromParent(); 300e058d276559bcecb11662c2548b249876e9430b0Craig Topper return; 301e058d276559bcecb11662c2548b249876e9430b0Craig Topper } 302e058d276559bcecb11662c2548b249876e9430b0Craig Topper 3037325f06051bab14196ab3701d515f5b110cefe26Chandler Carruth std::string Name = CI->getName().str(); 3047325f06051bab14196ab3701d515f5b110cefe26Chandler Carruth CI->setName(Name + ".old"); 3053c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem 306ccbf1e36d34c2d66600748bdd8767dc122629a58Chandler Carruth switch (NewFn->getIntrinsicID()) { 307ccbf1e36d34c2d66600748bdd8767dc122629a58Chandler Carruth default: 308b5dd9de7240f3018dcd8ce84c158547a5e0f1131Chris Lattner llvm_unreachable("Unknown function for CallInst upgrade."); 309ccbf1e36d34c2d66600748bdd8767dc122629a58Chandler Carruth 310ccbf1e36d34c2d66600748bdd8767dc122629a58Chandler Carruth case Intrinsic::ctlz: 31123e75da7e0622528be3c3908b5fe3ae8857cdf65Nuno Lopes case Intrinsic::cttz: 312ccbf1e36d34c2d66600748bdd8767dc122629a58Chandler Carruth assert(CI->getNumArgOperands() == 1 && 313ccbf1e36d34c2d66600748bdd8767dc122629a58Chandler Carruth "Mismatch between function args and call args"); 314ccbf1e36d34c2d66600748bdd8767dc122629a58Chandler Carruth CI->replaceAllUsesWith(Builder.CreateCall2(NewFn, CI->getArgOperand(0), 315ccbf1e36d34c2d66600748bdd8767dc122629a58Chandler Carruth Builder.getFalse(), Name)); 316ccbf1e36d34c2d66600748bdd8767dc122629a58Chandler Carruth CI->eraseFromParent(); 317ccbf1e36d34c2d66600748bdd8767dc122629a58Chandler Carruth return; 3183c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem 31906a6a300c5f7100e4665667c689369e078d2ad59Joel Jones case Intrinsic::arm_neon_vclz: { 32006a6a300c5f7100e4665667c689369e078d2ad59Joel Jones // Change name from llvm.arm.neon.vclz.* to llvm.ctlz.* 32106a6a300c5f7100e4665667c689369e078d2ad59Joel Jones CI->replaceAllUsesWith(Builder.CreateCall2(NewFn, CI->getArgOperand(0), 3227c82e6a32a84e238c9e4e57dd43eaba540a79ce1Joel Jones Builder.getFalse(), 32306a6a300c5f7100e4665667c689369e078d2ad59Joel Jones "llvm.ctlz." + Name.substr(14))); 32406a6a300c5f7100e4665667c689369e078d2ad59Joel Jones CI->eraseFromParent(); 32506a6a300c5f7100e4665667c689369e078d2ad59Joel Jones return; 32606a6a300c5f7100e4665667c689369e078d2ad59Joel Jones } 3277c82e6a32a84e238c9e4e57dd43eaba540a79ce1Joel Jones case Intrinsic::ctpop: { 3287c82e6a32a84e238c9e4e57dd43eaba540a79ce1Joel Jones CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(0))); 3297c82e6a32a84e238c9e4e57dd43eaba540a79ce1Joel Jones CI->eraseFromParent(); 3307c82e6a32a84e238c9e4e57dd43eaba540a79ce1Joel Jones return; 3317c82e6a32a84e238c9e4e57dd43eaba540a79ce1Joel Jones } 33206a6a300c5f7100e4665667c689369e078d2ad59Joel Jones 333cc95b57d42a4af1cbb0a0e4a4efc2133116dd21cCraig Topper case Intrinsic::x86_xop_vfrcz_ss: 334cc95b57d42a4af1cbb0a0e4a4efc2133116dd21cCraig Topper case Intrinsic::x86_xop_vfrcz_sd: 335cc95b57d42a4af1cbb0a0e4a4efc2133116dd21cCraig Topper CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(1), 336cc95b57d42a4af1cbb0a0e4a4efc2133116dd21cCraig Topper Name)); 337cc95b57d42a4af1cbb0a0e4a4efc2133116dd21cCraig Topper CI->eraseFromParent(); 338cc95b57d42a4af1cbb0a0e4a4efc2133116dd21cCraig Topper return; 339cc95b57d42a4af1cbb0a0e4a4efc2133116dd21cCraig Topper 3403c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem case Intrinsic::x86_sse41_ptestc: 3413c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem case Intrinsic::x86_sse41_ptestz: 342cc95b57d42a4af1cbb0a0e4a4efc2133116dd21cCraig Topper case Intrinsic::x86_sse41_ptestnzc: { 3433c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem // The arguments for these intrinsics used to be v4f32, and changed 3443c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem // to v2i64. This is purely a nop, since those are bitwise intrinsics. 3453c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem // So, the only thing required is a bitcast for both arguments. 3463c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem // First, check the arguments have the old type. 3473c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem Value *Arg0 = CI->getArgOperand(0); 3483c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem if (Arg0->getType() != VectorType::get(Type::getFloatTy(C), 4)) 3493c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem return; 3503c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem 3513c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem // Old intrinsic, add bitcasts 3523c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem Value *Arg1 = CI->getArgOperand(1); 3533c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem 3543c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem Value *BC0 = 3553c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem Builder.CreateBitCast(Arg0, 3563c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem VectorType::get(Type::getInt64Ty(C), 2), 3573c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem "cast"); 3583c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem Value *BC1 = 3593c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem Builder.CreateBitCast(Arg1, 3603c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem VectorType::get(Type::getInt64Ty(C), 2), 3613c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem "cast"); 3623c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem 3633c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem CallInst* NewCall = Builder.CreateCall2(NewFn, BC0, BC1, Name); 3643c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem CI->replaceAllUsesWith(NewCall); 3653c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem CI->eraseFromParent(); 3663c98ce242e4b0f24e6b0dbcf47c5e06c54038419Nadav Rotem return; 367f9b83fcf95e8a280d7b117af8858596fe5b10d94Evan Cheng } 368cc95b57d42a4af1cbb0a0e4a4efc2133116dd21cCraig Topper } 3696994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth} 3706994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth 3716994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth// This tests each Function to determine if it needs upgrading. When we find 3726994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth// one we are interested in, we then upgrade all calls to reflect the new 3736994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth// function. 3746994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruthvoid llvm::UpgradeCallsToIntrinsic(Function* F) { 3756994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth assert(F && "Illegal attempt to upgrade a non-existent intrinsic."); 3766994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth 3776994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth // Upgrade the function and check if it is a totaly new function. 378b85e4eba85a38698f3b3332f82554bf8442547e2Chris Lattner Function *NewFn; 379f9b83fcf95e8a280d7b117af8858596fe5b10d94Evan Cheng if (UpgradeIntrinsicFunction(F, NewFn)) { 3806994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth if (NewFn != F) { 3816994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth // Replace all uses to the old function with the new one if necessary. 3826994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth for (Value::use_iterator UI = F->use_begin(), UE = F->use_end(); 3836994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth UI != UE; ) { 384b85e4eba85a38698f3b3332f82554bf8442547e2Chris Lattner if (CallInst *CI = dyn_cast<CallInst>(*UI++)) 3856994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth UpgradeIntrinsicCall(CI, NewFn); 3866994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth } 3876994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth // Remove old function, no longer used, from the module. 3886994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth F->eraseFromParent(); 3896994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth } 3906994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth } 3916994040a952e5fb27605eb3cf29ed86c4e59cf62Chandler Carruth} 392e4b275610a7a05b7ee4c0378a906a6330e4c4ab0Devang Patel 393