AutoUpgrade.cpp revision ebe69fe11e48d322045d5949c83283927a0d790b
1//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the auto-upgrade helper functions 11// 12//===----------------------------------------------------------------------===// 13 14#include "llvm/IR/AutoUpgrade.h" 15#include "llvm/IR/CFG.h" 16#include "llvm/IR/CallSite.h" 17#include "llvm/IR/Constants.h" 18#include "llvm/IR/DIBuilder.h" 19#include "llvm/IR/DebugInfo.h" 20#include "llvm/IR/DiagnosticInfo.h" 21#include "llvm/IR/Function.h" 22#include "llvm/IR/IRBuilder.h" 23#include "llvm/IR/Instruction.h" 24#include "llvm/IR/IntrinsicInst.h" 25#include "llvm/IR/LLVMContext.h" 26#include "llvm/IR/Module.h" 27#include "llvm/Support/ErrorHandling.h" 28#include <cstring> 29using namespace llvm; 30 31// Upgrade the declarations of the SSE4.1 functions whose arguments have 32// changed their type from v4f32 to v2i64. 33static bool UpgradeSSE41Function(Function* F, Intrinsic::ID IID, 34 Function *&NewFn) { 35 // Check whether this is an old version of the function, which received 36 // v4f32 arguments. 37 Type *Arg0Type = F->getFunctionType()->getParamType(0); 38 if (Arg0Type != VectorType::get(Type::getFloatTy(F->getContext()), 4)) 39 return false; 40 41 // Yes, it's old, replace it with new version. 42 F->setName(F->getName() + ".old"); 43 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); 44 return true; 45} 46 47// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask 48// arguments have changed their type from i32 to i8. 49static bool UpgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, 50 Function *&NewFn) { 51 // Check that the last argument is an i32. 52 Type *LastArgType = F->getFunctionType()->getParamType( 53 F->getFunctionType()->getNumParams() - 1); 54 if (!LastArgType->isIntegerTy(32)) 55 return false; 56 57 // Move this function aside and map down. 58 F->setName(F->getName() + ".old"); 59 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); 60 return true; 61} 62 63// Upgrade the declarations of AVX-512 cmp intrinsic functions whose 8-bit 64// immediates have changed their type from i32 to i8. 65static bool UpgradeAVX512CmpIntrinsic(Function *F, Intrinsic::ID IID, 66 Function *&NewFn) { 67 // Check that the last argument is an i32. 68 Type *LastArgType = F->getFunctionType()->getParamType(2); 69 if (!LastArgType->isIntegerTy(32)) 70 return false; 71 72 // Move this function aside and map down. 73 F->setName(F->getName() + ".old"); 74 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); 75 return true; 76} 77 78static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { 79 assert(F && "Illegal to upgrade a non-existent Function."); 80 81 // Quickly eliminate it, if it's not a candidate. 82 StringRef Name = F->getName(); 83 if (Name.size() <= 8 || !Name.startswith("llvm.")) 84 return false; 85 Name = Name.substr(5); // Strip off "llvm." 86 87 switch (Name[0]) { 88 default: break; 89 case 'a': { 90 if (Name.startswith("arm.neon.vclz")) { 91 Type* args[2] = { 92 F->arg_begin()->getType(), 93 Type::getInt1Ty(F->getContext()) 94 }; 95 // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to 96 // the end of the name. Change name from llvm.arm.neon.vclz.* to 97 // llvm.ctlz.* 98 FunctionType* fType = FunctionType::get(F->getReturnType(), args, false); 99 NewFn = Function::Create(fType, F->getLinkage(), 100 "llvm.ctlz." + Name.substr(14), F->getParent()); 101 return true; 102 } 103 if (Name.startswith("arm.neon.vcnt")) { 104 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop, 105 F->arg_begin()->getType()); 106 return true; 107 } 108 break; 109 } 110 case 'c': { 111 if (Name.startswith("ctlz.") && F->arg_size() == 1) { 112 F->setName(Name + ".old"); 113 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz, 114 F->arg_begin()->getType()); 115 return true; 116 } 117 if (Name.startswith("cttz.") && F->arg_size() == 1) { 118 F->setName(Name + ".old"); 119 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz, 120 F->arg_begin()->getType()); 121 return true; 122 } 123 break; 124 } 125 case 'd': { 126 if (Name.startswith("dbg.declare") && F->arg_size() == 2) { 127 F->setName(Name + ".old"); 128 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_declare); 129 return true; 130 } 131 if (Name.startswith("dbg.value") && F->arg_size() == 3) { 132 F->setName(Name + ".old"); 133 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value); 134 return true; 135 } 136 break; 137 } 138 139 case 'o': 140 // We only need to change the name to match the mangling including the 141 // address space. 142 if (F->arg_size() == 2 && Name.startswith("objectsize.")) { 143 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() }; 144 if (F->getName() != Intrinsic::getName(Intrinsic::objectsize, Tys)) { 145 F->setName(Name + ".old"); 146 NewFn = Intrinsic::getDeclaration(F->getParent(), 147 Intrinsic::objectsize, Tys); 148 return true; 149 } 150 } 151 break; 152 153 case 'x': { 154 if (Name.startswith("x86.sse2.pcmpeq.") || 155 Name.startswith("x86.sse2.pcmpgt.") || 156 Name.startswith("x86.avx2.pcmpeq.") || 157 Name.startswith("x86.avx2.pcmpgt.") || 158 Name.startswith("x86.avx.vpermil.") || 159 Name == "x86.avx.movnt.dq.256" || 160 Name == "x86.avx.movnt.pd.256" || 161 Name == "x86.avx.movnt.ps.256" || 162 Name == "x86.sse42.crc32.64.8" || 163 Name == "x86.avx.vbroadcast.ss" || 164 Name == "x86.avx.vbroadcast.ss.256" || 165 Name == "x86.avx.vbroadcast.sd.256" || 166 Name == "x86.sse2.psll.dq" || 167 Name == "x86.sse2.psrl.dq" || 168 Name == "x86.avx2.psll.dq" || 169 Name == "x86.avx2.psrl.dq" || 170 Name == "x86.sse2.psll.dq.bs" || 171 Name == "x86.sse2.psrl.dq.bs" || 172 Name == "x86.avx2.psll.dq.bs" || 173 Name == "x86.avx2.psrl.dq.bs" || 174 (Name.startswith("x86.xop.vpcom") && F->arg_size() == 2)) { 175 NewFn = nullptr; 176 return true; 177 } 178 // SSE4.1 ptest functions may have an old signature. 179 if (Name.startswith("x86.sse41.ptest")) { 180 if (Name == "x86.sse41.ptestc") 181 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestc, NewFn); 182 if (Name == "x86.sse41.ptestz") 183 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestz, NewFn); 184 if (Name == "x86.sse41.ptestnzc") 185 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestnzc, NewFn); 186 } 187 // Several blend and other instructions with maskes used the wrong number of 188 // bits. 189 if (Name == "x86.sse41.pblendw") 190 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_pblendw, 191 NewFn); 192 if (Name == "x86.sse41.blendpd") 193 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_blendpd, 194 NewFn); 195 if (Name == "x86.sse41.blendps") 196 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_blendps, 197 NewFn); 198 if (Name == "x86.sse41.insertps") 199 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_insertps, 200 NewFn); 201 if (Name == "x86.sse41.dppd") 202 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dppd, 203 NewFn); 204 if (Name == "x86.sse41.dpps") 205 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dpps, 206 NewFn); 207 if (Name == "x86.sse41.mpsadbw") 208 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_mpsadbw, 209 NewFn); 210 if (Name == "x86.avx.blend.pd.256") 211 return UpgradeX86IntrinsicsWith8BitMask( 212 F, Intrinsic::x86_avx_blend_pd_256, NewFn); 213 if (Name == "x86.avx.blend.ps.256") 214 return UpgradeX86IntrinsicsWith8BitMask( 215 F, Intrinsic::x86_avx_blend_ps_256, NewFn); 216 if (Name == "x86.avx.dp.ps.256") 217 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx_dp_ps_256, 218 NewFn); 219 if (Name == "x86.avx2.pblendw") 220 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_pblendw, 221 NewFn); 222 if (Name == "x86.avx2.pblendd.128") 223 return UpgradeX86IntrinsicsWith8BitMask( 224 F, Intrinsic::x86_avx2_pblendd_128, NewFn); 225 if (Name == "x86.avx2.pblendd.256") 226 return UpgradeX86IntrinsicsWith8BitMask( 227 F, Intrinsic::x86_avx2_pblendd_256, NewFn); 228 if (Name == "x86.avx2.mpsadbw") 229 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw, 230 NewFn); 231 232 if (Name == "x86.avx512.mask.cmp.ps.512") 233 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_ps_512, 234 NewFn); 235 if (Name == "x86.avx512.mask.cmp.pd.512") 236 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_pd_512, 237 NewFn); 238 239 if (Name == "x86.avx512.mask.cmp.b.512") 240 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_b_512, 241 NewFn); 242 if (Name == "x86.avx512.mask.cmp.w.512") 243 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_w_512, 244 NewFn); 245 if (Name == "x86.avx512.mask.cmp.d.512") 246 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_d_512, 247 NewFn); 248 if (Name == "x86.avx512.mask.cmp.q.512") 249 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_q_512, 250 NewFn); 251 if (Name == "x86.avx512.mask.ucmp.b.512") 252 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_b_512, 253 NewFn); 254 if (Name == "x86.avx512.mask.ucmp.w.512") 255 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_w_512, 256 NewFn); 257 if (Name == "x86.avx512.mask.ucmp.d.512") 258 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_d_512, 259 NewFn); 260 if (Name == "x86.avx512.mask.ucmp.q.512") 261 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_q_512, 262 NewFn); 263 264 if (Name == "x86.avx512.mask.cmp.b.256") 265 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_b_256, 266 NewFn); 267 if (Name == "x86.avx512.mask.cmp.w.256") 268 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_w_256, 269 NewFn); 270 if (Name == "x86.avx512.mask.cmp.d.256") 271 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_d_256, 272 NewFn); 273 if (Name == "x86.avx512.mask.cmp.q.256") 274 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_q_256, 275 NewFn); 276 if (Name == "x86.avx512.mask.ucmp.b.256") 277 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_b_256, 278 NewFn); 279 if (Name == "x86.avx512.mask.ucmp.w.256") 280 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_w_256, 281 NewFn); 282 if (Name == "x86.avx512.mask.ucmp.d.256") 283 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_d_256, 284 NewFn); 285 if (Name == "x86.avx512.mask.ucmp.q.256") 286 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_q_256, 287 NewFn); 288 289 if (Name == "x86.avx512.mask.cmp.b.128") 290 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_b_128, 291 NewFn); 292 if (Name == "x86.avx512.mask.cmp.w.128") 293 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_w_128, 294 NewFn); 295 if (Name == "x86.avx512.mask.cmp.d.128") 296 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_d_128, 297 NewFn); 298 if (Name == "x86.avx512.mask.cmp.q.128") 299 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_q_128, 300 NewFn); 301 if (Name == "x86.avx512.mask.ucmp.b.128") 302 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_b_128, 303 NewFn); 304 if (Name == "x86.avx512.mask.ucmp.w.128") 305 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_w_128, 306 NewFn); 307 if (Name == "x86.avx512.mask.ucmp.d.128") 308 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_d_128, 309 NewFn); 310 if (Name == "x86.avx512.mask.ucmp.q.128") 311 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_q_128, 312 NewFn); 313 314 // frcz.ss/sd may need to have an argument dropped 315 if (Name.startswith("x86.xop.vfrcz.ss") && F->arg_size() == 2) { 316 F->setName(Name + ".old"); 317 NewFn = Intrinsic::getDeclaration(F->getParent(), 318 Intrinsic::x86_xop_vfrcz_ss); 319 return true; 320 } 321 if (Name.startswith("x86.xop.vfrcz.sd") && F->arg_size() == 2) { 322 F->setName(Name + ".old"); 323 NewFn = Intrinsic::getDeclaration(F->getParent(), 324 Intrinsic::x86_xop_vfrcz_sd); 325 return true; 326 } 327 // Fix the FMA4 intrinsics to remove the 4 328 if (Name.startswith("x86.fma4.")) { 329 F->setName("llvm.x86.fma" + Name.substr(8)); 330 NewFn = F; 331 return true; 332 } 333 break; 334 } 335 } 336 337 // This may not belong here. This function is effectively being overloaded 338 // to both detect an intrinsic which needs upgrading, and to provide the 339 // upgraded form of the intrinsic. We should perhaps have two separate 340 // functions for this. 341 return false; 342} 343 344bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) { 345 NewFn = nullptr; 346 bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn); 347 348 // Upgrade intrinsic attributes. This does not change the function. 349 if (NewFn) 350 F = NewFn; 351 if (unsigned id = F->getIntrinsicID()) 352 F->setAttributes(Intrinsic::getAttributes(F->getContext(), 353 (Intrinsic::ID)id)); 354 return Upgraded; 355} 356 357bool llvm::UpgradeGlobalVariable(GlobalVariable *GV) { 358 // Nothing to do yet. 359 return false; 360} 361 362static MDNode *getNodeField(const MDNode *DbgNode, unsigned Elt) { 363 if (!DbgNode || Elt >= DbgNode->getNumOperands()) 364 return nullptr; 365 return dyn_cast_or_null<MDNode>(DbgNode->getOperand(Elt)); 366} 367 368static MetadataAsValue *getExpression(Value *VarOperand, Function *F) { 369 // Old-style DIVariables have an optional expression as the 8th element. 370 DIExpression Expr(getNodeField( 371 cast<MDNode>(cast<MetadataAsValue>(VarOperand)->getMetadata()), 8)); 372 if (!Expr) { 373 DIBuilder DIB(*F->getParent(), /*AllowUnresolved*/ false); 374 Expr = DIB.createExpression(); 375 } 376 return MetadataAsValue::get(F->getContext(), Expr); 377} 378 379// Handles upgrading SSE2 and AVX2 PSLLDQ intrinsics by converting them 380// to byte shuffles. 381static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, LLVMContext &C, 382 Value *Op, unsigned NumLanes, 383 unsigned Shift) { 384 // Each lane is 16 bytes. 385 unsigned NumElts = NumLanes * 16; 386 387 // Bitcast from a 64-bit element type to a byte element type. 388 Op = Builder.CreateBitCast(Op, 389 VectorType::get(Type::getInt8Ty(C), NumElts), 390 "cast"); 391 // We'll be shuffling in zeroes. 392 Value *Res = ConstantVector::getSplat(NumElts, Builder.getInt8(0)); 393 394 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise, 395 // we'll just return the zero vector. 396 if (Shift < 16) { 397 SmallVector<Constant*, 32> Idxs; 398 // 256-bit version is split into two 16-byte lanes. 399 for (unsigned l = 0; l != NumElts; l += 16) 400 for (unsigned i = 0; i != 16; ++i) { 401 unsigned Idx = NumElts + i - Shift; 402 if (Idx < NumElts) 403 Idx -= NumElts - 16; // end of lane, switch operand. 404 Idxs.push_back(Builder.getInt32(Idx + l)); 405 } 406 407 Res = Builder.CreateShuffleVector(Res, Op, ConstantVector::get(Idxs)); 408 } 409 410 // Bitcast back to a 64-bit element type. 411 return Builder.CreateBitCast(Res, 412 VectorType::get(Type::getInt64Ty(C), 2*NumLanes), 413 "cast"); 414} 415 416// Handles upgrading SSE2 and AVX2 PSRLDQ intrinsics by converting them 417// to byte shuffles. 418static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, LLVMContext &C, 419 Value *Op, unsigned NumLanes, 420 unsigned Shift) { 421 // Each lane is 16 bytes. 422 unsigned NumElts = NumLanes * 16; 423 424 // Bitcast from a 64-bit element type to a byte element type. 425 Op = Builder.CreateBitCast(Op, 426 VectorType::get(Type::getInt8Ty(C), NumElts), 427 "cast"); 428 // We'll be shuffling in zeroes. 429 Value *Res = ConstantVector::getSplat(NumElts, Builder.getInt8(0)); 430 431 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise, 432 // we'll just return the zero vector. 433 if (Shift < 16) { 434 SmallVector<Constant*, 32> Idxs; 435 // 256-bit version is split into two 16-byte lanes. 436 for (unsigned l = 0; l != NumElts; l += 16) 437 for (unsigned i = 0; i != 16; ++i) { 438 unsigned Idx = i + Shift; 439 if (Idx >= 16) 440 Idx += NumElts - 16; // end of lane, switch operand. 441 Idxs.push_back(Builder.getInt32(Idx + l)); 442 } 443 444 Res = Builder.CreateShuffleVector(Op, Res, ConstantVector::get(Idxs)); 445 } 446 447 // Bitcast back to a 64-bit element type. 448 return Builder.CreateBitCast(Res, 449 VectorType::get(Type::getInt64Ty(C), 2*NumLanes), 450 "cast"); 451} 452 453// UpgradeIntrinsicCall - Upgrade a call to an old intrinsic to be a call the 454// upgraded intrinsic. All argument and return casting must be provided in 455// order to seamlessly integrate with existing context. 456void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { 457 Function *F = CI->getCalledFunction(); 458 LLVMContext &C = CI->getContext(); 459 IRBuilder<> Builder(C); 460 Builder.SetInsertPoint(CI->getParent(), CI); 461 462 assert(F && "Intrinsic call is not direct?"); 463 464 if (!NewFn) { 465 // Get the Function's name. 466 StringRef Name = F->getName(); 467 468 Value *Rep; 469 // Upgrade packed integer vector compares intrinsics to compare instructions 470 if (Name.startswith("llvm.x86.sse2.pcmpeq.") || 471 Name.startswith("llvm.x86.avx2.pcmpeq.")) { 472 Rep = Builder.CreateICmpEQ(CI->getArgOperand(0), CI->getArgOperand(1), 473 "pcmpeq"); 474 // need to sign extend since icmp returns vector of i1 475 Rep = Builder.CreateSExt(Rep, CI->getType(), ""); 476 } else if (Name.startswith("llvm.x86.sse2.pcmpgt.") || 477 Name.startswith("llvm.x86.avx2.pcmpgt.")) { 478 Rep = Builder.CreateICmpSGT(CI->getArgOperand(0), CI->getArgOperand(1), 479 "pcmpgt"); 480 // need to sign extend since icmp returns vector of i1 481 Rep = Builder.CreateSExt(Rep, CI->getType(), ""); 482 } else if (Name == "llvm.x86.avx.movnt.dq.256" || 483 Name == "llvm.x86.avx.movnt.ps.256" || 484 Name == "llvm.x86.avx.movnt.pd.256") { 485 IRBuilder<> Builder(C); 486 Builder.SetInsertPoint(CI->getParent(), CI); 487 488 Module *M = F->getParent(); 489 SmallVector<Metadata *, 1> Elts; 490 Elts.push_back( 491 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1))); 492 MDNode *Node = MDNode::get(C, Elts); 493 494 Value *Arg0 = CI->getArgOperand(0); 495 Value *Arg1 = CI->getArgOperand(1); 496 497 // Convert the type of the pointer to a pointer to the stored type. 498 Value *BC = Builder.CreateBitCast(Arg0, 499 PointerType::getUnqual(Arg1->getType()), 500 "cast"); 501 StoreInst *SI = Builder.CreateStore(Arg1, BC); 502 SI->setMetadata(M->getMDKindID("nontemporal"), Node); 503 SI->setAlignment(16); 504 505 // Remove intrinsic. 506 CI->eraseFromParent(); 507 return; 508 } else if (Name.startswith("llvm.x86.xop.vpcom")) { 509 Intrinsic::ID intID; 510 if (Name.endswith("ub")) 511 intID = Intrinsic::x86_xop_vpcomub; 512 else if (Name.endswith("uw")) 513 intID = Intrinsic::x86_xop_vpcomuw; 514 else if (Name.endswith("ud")) 515 intID = Intrinsic::x86_xop_vpcomud; 516 else if (Name.endswith("uq")) 517 intID = Intrinsic::x86_xop_vpcomuq; 518 else if (Name.endswith("b")) 519 intID = Intrinsic::x86_xop_vpcomb; 520 else if (Name.endswith("w")) 521 intID = Intrinsic::x86_xop_vpcomw; 522 else if (Name.endswith("d")) 523 intID = Intrinsic::x86_xop_vpcomd; 524 else if (Name.endswith("q")) 525 intID = Intrinsic::x86_xop_vpcomq; 526 else 527 llvm_unreachable("Unknown suffix"); 528 529 Name = Name.substr(18); // strip off "llvm.x86.xop.vpcom" 530 unsigned Imm; 531 if (Name.startswith("lt")) 532 Imm = 0; 533 else if (Name.startswith("le")) 534 Imm = 1; 535 else if (Name.startswith("gt")) 536 Imm = 2; 537 else if (Name.startswith("ge")) 538 Imm = 3; 539 else if (Name.startswith("eq")) 540 Imm = 4; 541 else if (Name.startswith("ne")) 542 Imm = 5; 543 else if (Name.startswith("false")) 544 Imm = 6; 545 else if (Name.startswith("true")) 546 Imm = 7; 547 else 548 llvm_unreachable("Unknown condition"); 549 550 Function *VPCOM = Intrinsic::getDeclaration(F->getParent(), intID); 551 Rep = Builder.CreateCall3(VPCOM, CI->getArgOperand(0), 552 CI->getArgOperand(1), Builder.getInt8(Imm)); 553 } else if (Name == "llvm.x86.sse42.crc32.64.8") { 554 Function *CRC32 = Intrinsic::getDeclaration(F->getParent(), 555 Intrinsic::x86_sse42_crc32_32_8); 556 Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C)); 557 Rep = Builder.CreateCall2(CRC32, Trunc0, CI->getArgOperand(1)); 558 Rep = Builder.CreateZExt(Rep, CI->getType(), ""); 559 } else if (Name.startswith("llvm.x86.avx.vbroadcast")) { 560 // Replace broadcasts with a series of insertelements. 561 Type *VecTy = CI->getType(); 562 Type *EltTy = VecTy->getVectorElementType(); 563 unsigned EltNum = VecTy->getVectorNumElements(); 564 Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0), 565 EltTy->getPointerTo()); 566 Value *Load = Builder.CreateLoad(Cast); 567 Type *I32Ty = Type::getInt32Ty(C); 568 Rep = UndefValue::get(VecTy); 569 for (unsigned I = 0; I < EltNum; ++I) 570 Rep = Builder.CreateInsertElement(Rep, Load, 571 ConstantInt::get(I32Ty, I)); 572 } else if (Name == "llvm.x86.sse2.psll.dq") { 573 // 128-bit shift left specified in bits. 574 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 575 Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1, 576 Shift / 8); // Shift is in bits. 577 } else if (Name == "llvm.x86.sse2.psrl.dq") { 578 // 128-bit shift right specified in bits. 579 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 580 Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1, 581 Shift / 8); // Shift is in bits. 582 } else if (Name == "llvm.x86.avx2.psll.dq") { 583 // 256-bit shift left specified in bits. 584 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 585 Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2, 586 Shift / 8); // Shift is in bits. 587 } else if (Name == "llvm.x86.avx2.psrl.dq") { 588 // 256-bit shift right specified in bits. 589 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 590 Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2, 591 Shift / 8); // Shift is in bits. 592 } else if (Name == "llvm.x86.sse2.psll.dq.bs") { 593 // 128-bit shift left specified in bytes. 594 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 595 Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1, 596 Shift); 597 } else if (Name == "llvm.x86.sse2.psrl.dq.bs") { 598 // 128-bit shift right specified in bytes. 599 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 600 Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1, 601 Shift); 602 } else if (Name == "llvm.x86.avx2.psll.dq.bs") { 603 // 256-bit shift left specified in bytes. 604 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 605 Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2, 606 Shift); 607 } else if (Name == "llvm.x86.avx2.psrl.dq.bs") { 608 // 256-bit shift right specified in bytes. 609 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 610 Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2, 611 Shift); 612 } else { 613 bool PD128 = false, PD256 = false, PS128 = false, PS256 = false; 614 if (Name == "llvm.x86.avx.vpermil.pd.256") 615 PD256 = true; 616 else if (Name == "llvm.x86.avx.vpermil.pd") 617 PD128 = true; 618 else if (Name == "llvm.x86.avx.vpermil.ps.256") 619 PS256 = true; 620 else if (Name == "llvm.x86.avx.vpermil.ps") 621 PS128 = true; 622 623 if (PD256 || PD128 || PS256 || PS128) { 624 Value *Op0 = CI->getArgOperand(0); 625 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 626 SmallVector<Constant*, 8> Idxs; 627 628 if (PD128) 629 for (unsigned i = 0; i != 2; ++i) 630 Idxs.push_back(Builder.getInt32((Imm >> i) & 0x1)); 631 else if (PD256) 632 for (unsigned l = 0; l != 4; l+=2) 633 for (unsigned i = 0; i != 2; ++i) 634 Idxs.push_back(Builder.getInt32(((Imm >> (l+i)) & 0x1) + l)); 635 else if (PS128) 636 for (unsigned i = 0; i != 4; ++i) 637 Idxs.push_back(Builder.getInt32((Imm >> (2 * i)) & 0x3)); 638 else if (PS256) 639 for (unsigned l = 0; l != 8; l+=4) 640 for (unsigned i = 0; i != 4; ++i) 641 Idxs.push_back(Builder.getInt32(((Imm >> (2 * i)) & 0x3) + l)); 642 else 643 llvm_unreachable("Unexpected function"); 644 645 Rep = Builder.CreateShuffleVector(Op0, Op0, ConstantVector::get(Idxs)); 646 } else { 647 llvm_unreachable("Unknown function for CallInst upgrade."); 648 } 649 } 650 651 CI->replaceAllUsesWith(Rep); 652 CI->eraseFromParent(); 653 return; 654 } 655 656 std::string Name = CI->getName().str(); 657 if (!Name.empty()) 658 CI->setName(Name + ".old"); 659 660 switch (NewFn->getIntrinsicID()) { 661 default: 662 llvm_unreachable("Unknown function for CallInst upgrade."); 663 664 // Upgrade debug intrinsics to use an additional DIExpression argument. 665 case Intrinsic::dbg_declare: { 666 auto NewCI = 667 Builder.CreateCall3(NewFn, CI->getArgOperand(0), CI->getArgOperand(1), 668 getExpression(CI->getArgOperand(1), F), Name); 669 NewCI->setDebugLoc(CI->getDebugLoc()); 670 CI->replaceAllUsesWith(NewCI); 671 CI->eraseFromParent(); 672 return; 673 } 674 case Intrinsic::dbg_value: { 675 auto NewCI = Builder.CreateCall4( 676 NewFn, CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), 677 getExpression(CI->getArgOperand(2), F), Name); 678 NewCI->setDebugLoc(CI->getDebugLoc()); 679 CI->replaceAllUsesWith(NewCI); 680 CI->eraseFromParent(); 681 return; 682 } 683 case Intrinsic::ctlz: 684 case Intrinsic::cttz: 685 assert(CI->getNumArgOperands() == 1 && 686 "Mismatch between function args and call args"); 687 CI->replaceAllUsesWith(Builder.CreateCall2(NewFn, CI->getArgOperand(0), 688 Builder.getFalse(), Name)); 689 CI->eraseFromParent(); 690 return; 691 692 case Intrinsic::objectsize: 693 CI->replaceAllUsesWith(Builder.CreateCall2(NewFn, 694 CI->getArgOperand(0), 695 CI->getArgOperand(1), 696 Name)); 697 CI->eraseFromParent(); 698 return; 699 700 case Intrinsic::ctpop: { 701 CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(0))); 702 CI->eraseFromParent(); 703 return; 704 } 705 706 case Intrinsic::x86_xop_vfrcz_ss: 707 case Intrinsic::x86_xop_vfrcz_sd: 708 CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(1), 709 Name)); 710 CI->eraseFromParent(); 711 return; 712 713 case Intrinsic::x86_sse41_ptestc: 714 case Intrinsic::x86_sse41_ptestz: 715 case Intrinsic::x86_sse41_ptestnzc: { 716 // The arguments for these intrinsics used to be v4f32, and changed 717 // to v2i64. This is purely a nop, since those are bitwise intrinsics. 718 // So, the only thing required is a bitcast for both arguments. 719 // First, check the arguments have the old type. 720 Value *Arg0 = CI->getArgOperand(0); 721 if (Arg0->getType() != VectorType::get(Type::getFloatTy(C), 4)) 722 return; 723 724 // Old intrinsic, add bitcasts 725 Value *Arg1 = CI->getArgOperand(1); 726 727 Value *BC0 = 728 Builder.CreateBitCast(Arg0, 729 VectorType::get(Type::getInt64Ty(C), 2), 730 "cast"); 731 Value *BC1 = 732 Builder.CreateBitCast(Arg1, 733 VectorType::get(Type::getInt64Ty(C), 2), 734 "cast"); 735 736 CallInst* NewCall = Builder.CreateCall2(NewFn, BC0, BC1, Name); 737 CI->replaceAllUsesWith(NewCall); 738 CI->eraseFromParent(); 739 return; 740 } 741 742 case Intrinsic::x86_sse41_pblendw: 743 case Intrinsic::x86_sse41_blendpd: 744 case Intrinsic::x86_sse41_blendps: 745 case Intrinsic::x86_sse41_insertps: 746 case Intrinsic::x86_sse41_dppd: 747 case Intrinsic::x86_sse41_dpps: 748 case Intrinsic::x86_sse41_mpsadbw: 749 case Intrinsic::x86_avx_blend_pd_256: 750 case Intrinsic::x86_avx_blend_ps_256: 751 case Intrinsic::x86_avx_dp_ps_256: 752 case Intrinsic::x86_avx2_pblendw: 753 case Intrinsic::x86_avx2_pblendd_128: 754 case Intrinsic::x86_avx2_pblendd_256: 755 case Intrinsic::x86_avx2_mpsadbw: { 756 // Need to truncate the last argument from i32 to i8 -- this argument models 757 // an inherently 8-bit immediate operand to these x86 instructions. 758 SmallVector<Value *, 4> Args(CI->arg_operands().begin(), 759 CI->arg_operands().end()); 760 761 // Replace the last argument with a trunc. 762 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc"); 763 764 CallInst *NewCall = Builder.CreateCall(NewFn, Args); 765 CI->replaceAllUsesWith(NewCall); 766 CI->eraseFromParent(); 767 return; 768 } 769 case Intrinsic::x86_avx512_mask_cmp_ps_512: 770 case Intrinsic::x86_avx512_mask_cmp_pd_512: { 771 // Need to truncate the last argument from i32 to i8 -- this argument models 772 // an inherently 8-bit immediate operand to these x86 instructions. 773 SmallVector<Value *, 5> Args(CI->arg_operands().begin(), 774 CI->arg_operands().end()); 775 776 // Replace the last argument with a trunc. 777 Args[2] = Builder.CreateTrunc(Args[2], Type::getInt8Ty(C), "trunc"); 778 779 CallInst *NewCall = Builder.CreateCall(NewFn, Args); 780 CI->replaceAllUsesWith(NewCall); 781 CI->eraseFromParent(); 782 return; 783 } 784 } 785} 786 787// This tests each Function to determine if it needs upgrading. When we find 788// one we are interested in, we then upgrade all calls to reflect the new 789// function. 790void llvm::UpgradeCallsToIntrinsic(Function* F) { 791 assert(F && "Illegal attempt to upgrade a non-existent intrinsic."); 792 793 // Upgrade the function and check if it is a totaly new function. 794 Function *NewFn; 795 if (UpgradeIntrinsicFunction(F, NewFn)) { 796 if (NewFn != F) { 797 // Replace all uses to the old function with the new one if necessary. 798 for (Value::user_iterator UI = F->user_begin(), UE = F->user_end(); 799 UI != UE; ) { 800 if (CallInst *CI = dyn_cast<CallInst>(*UI++)) 801 UpgradeIntrinsicCall(CI, NewFn); 802 } 803 // Remove old function, no longer used, from the module. 804 F->eraseFromParent(); 805 } 806 } 807} 808 809void llvm::UpgradeInstWithTBAATag(Instruction *I) { 810 MDNode *MD = I->getMetadata(LLVMContext::MD_tbaa); 811 assert(MD && "UpgradeInstWithTBAATag should have a TBAA tag"); 812 // Check if the tag uses struct-path aware TBAA format. 813 if (isa<MDNode>(MD->getOperand(0)) && MD->getNumOperands() >= 3) 814 return; 815 816 if (MD->getNumOperands() == 3) { 817 Metadata *Elts[] = {MD->getOperand(0), MD->getOperand(1)}; 818 MDNode *ScalarType = MDNode::get(I->getContext(), Elts); 819 // Create a MDNode <ScalarType, ScalarType, offset 0, const> 820 Metadata *Elts2[] = {ScalarType, ScalarType, 821 ConstantAsMetadata::get(Constant::getNullValue( 822 Type::getInt64Ty(I->getContext()))), 823 MD->getOperand(2)}; 824 I->setMetadata(LLVMContext::MD_tbaa, MDNode::get(I->getContext(), Elts2)); 825 } else { 826 // Create a MDNode <MD, MD, offset 0> 827 Metadata *Elts[] = {MD, MD, ConstantAsMetadata::get(Constant::getNullValue( 828 Type::getInt64Ty(I->getContext())))}; 829 I->setMetadata(LLVMContext::MD_tbaa, MDNode::get(I->getContext(), Elts)); 830 } 831} 832 833Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, 834 Instruction *&Temp) { 835 if (Opc != Instruction::BitCast) 836 return nullptr; 837 838 Temp = nullptr; 839 Type *SrcTy = V->getType(); 840 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() && 841 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) { 842 LLVMContext &Context = V->getContext(); 843 844 // We have no information about target data layout, so we assume that 845 // the maximum pointer size is 64bit. 846 Type *MidTy = Type::getInt64Ty(Context); 847 Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy); 848 849 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy); 850 } 851 852 return nullptr; 853} 854 855Value *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) { 856 if (Opc != Instruction::BitCast) 857 return nullptr; 858 859 Type *SrcTy = C->getType(); 860 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() && 861 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) { 862 LLVMContext &Context = C->getContext(); 863 864 // We have no information about target data layout, so we assume that 865 // the maximum pointer size is 64bit. 866 Type *MidTy = Type::getInt64Ty(Context); 867 868 return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy), 869 DestTy); 870 } 871 872 return nullptr; 873} 874 875/// Check the debug info version number, if it is out-dated, drop the debug 876/// info. Return true if module is modified. 877bool llvm::UpgradeDebugInfo(Module &M) { 878 unsigned Version = getDebugMetadataVersionFromModule(M); 879 if (Version == DEBUG_METADATA_VERSION) 880 return false; 881 882 bool RetCode = StripDebugInfo(M); 883 if (RetCode) { 884 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version); 885 M.getContext().diagnose(DiagVersion); 886 } 887 return RetCode; 888} 889 890void llvm::UpgradeMDStringConstant(std::string &String) { 891 const std::string OldPrefix = "llvm.vectorizer."; 892 if (String == "llvm.vectorizer.unroll") { 893 String = "llvm.loop.interleave.count"; 894 } else if (String.find(OldPrefix) == 0) { 895 String.replace(0, OldPrefix.size(), "llvm.loop.vectorize."); 896 } 897} 898