AutoUpgrade.cpp revision 804f034bd18789e9bbf4c70c10189dd6dbf04128 
1//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the auto-upgrade helper functions
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/AutoUpgrade.h"
15#include "llvm/IR/Constants.h"
16#include "llvm/IR/Function.h"
17#include "llvm/IR/IRBuilder.h"
18#include "llvm/IR/Instruction.h"
19#include "llvm/IR/IntrinsicInst.h"
20#include "llvm/IR/LLVMContext.h"
21#include "llvm/IR/Module.h"
22#include "llvm/Support/CFG.h"
23#include "llvm/Support/CallSite.h"
24#include "llvm/Support/ErrorHandling.h"
25#include <cstring>
26using namespace llvm;
27
28// Upgrade the declarations of the SSE4.1 functions whose arguments have
29// changed their type from v4f32 to v2i64.
30static bool UpgradeSSE41Function(Function* F, Intrinsic::ID IID,
31                                 Function *&NewFn) {
32  // Check whether this is an old version of the function, which received
33  // v4f32 arguments.
34  Type *Arg0Type = F->getFunctionType()->getParamType(0);
35  if (Arg0Type != VectorType::get(Type::getFloatTy(F->getContext()), 4))
36    return false;
37
38  // Yes, it's old, replace it with new version.
39  F->setName(F->getName() + ".old");
40  NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
41  return true;
42}
43
44static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
45  assert(F && "Illegal to upgrade a non-existent Function.");
46
47  // Quickly eliminate it, if it's not a candidate.
48  StringRef Name = F->getName();
49  if (Name.size() <= 8 || !Name.startswith("llvm."))
50    return false;
51  Name = Name.substr(5); // Strip off "llvm."
52
53  switch (Name[0]) {
54  default: break;
55  case 'a': {
56    if (Name.startswith("arm.neon.vclz")) {
57      Type* args[2] = {
58        F->arg_begin()->getType(),
59        Type::getInt1Ty(F->getContext())
60      };
61      // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to
62      // the end of the name. Change name from llvm.arm.neon.vclz.* to
63      //  llvm.ctlz.*
64      FunctionType* fType = FunctionType::get(F->getReturnType(), args, false);
65      NewFn = Function::Create(fType, F->getLinkage(),
66                               "llvm.ctlz." + Name.substr(14), F->getParent());
67      return true;
68    }
69    if (Name.startswith("arm.neon.vcnt")) {
70      NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
71                                        F->arg_begin()->getType());
72      return true;
73    }
74    break;
75  }
76  case 'c': {
77    if (Name.startswith("ctlz.") && F->arg_size() == 1) {
78      F->setName(Name + ".old");
79      NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
80                                        F->arg_begin()->getType());
81      return true;
82    }
83    if (Name.startswith("cttz.") && F->arg_size() == 1) {
84      F->setName(Name + ".old");
85      NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz,
86                                        F->arg_begin()->getType());
87      return true;
88    }
89    break;
90  }
91  case 'x': {
92    if (Name.startswith("x86.sse2.pcmpeq.") ||
93        Name.startswith("x86.sse2.pcmpgt.") ||
94        Name.startswith("x86.avx2.pcmpeq.") ||
95        Name.startswith("x86.avx2.pcmpgt.") ||
96        Name.startswith("x86.avx.vpermil.") ||
97        Name == "x86.avx.movnt.dq.256" ||
98        Name == "x86.avx.movnt.pd.256" ||
99        Name == "x86.avx.movnt.ps.256" ||
100        (Name.startswith("x86.xop.vpcom") && F->arg_size() == 2)) {
101      NewFn = 0;
102      return true;
103    }
104    // SSE4.1 ptest functions may have an old signature.
105    if (Name.startswith("x86.sse41.ptest")) {
106      if (Name == "x86.sse41.ptestc")
107        return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestc, NewFn);
108      if (Name == "x86.sse41.ptestz")
109        return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestz, NewFn);
110      if (Name == "x86.sse41.ptestnzc")
111        return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestnzc, NewFn);
112    }
113    // frcz.ss/sd may need to have an argument dropped
114    if (Name.startswith("x86.xop.vfrcz.ss") && F->arg_size() == 2) {
115      F->setName(Name + ".old");
116      NewFn = Intrinsic::getDeclaration(F->getParent(),
117                                        Intrinsic::x86_xop_vfrcz_ss);
118      return true;
119    }
120    if (Name.startswith("x86.xop.vfrcz.sd") && F->arg_size() == 2) {
121      F->setName(Name + ".old");
122      NewFn = Intrinsic::getDeclaration(F->getParent(),
123                                        Intrinsic::x86_xop_vfrcz_sd);
124      return true;
125    }
126    // Fix the FMA4 intrinsics to remove the 4
127    if (Name.startswith("x86.fma4.")) {
128      F->setName("llvm.x86.fma" + Name.substr(8));
129      NewFn = F;
130      return true;
131    }
132    break;
133  }
134  }
135
136  //  This may not belong here. This function is effectively being overloaded
137  //  to both detect an intrinsic which needs upgrading, and to provide the
138  //  upgraded form of the intrinsic. We should perhaps have two separate
139  //  functions for this.
140  return false;
141}
142
143bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) {
144  NewFn = 0;
145  bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn);
146
147  // Upgrade intrinsic attributes.  This does not change the function.
148  if (NewFn)
149    F = NewFn;
150  if (unsigned id = F->getIntrinsicID())
151    F->setAttributes(Intrinsic::getAttributes(F->getContext(),
152                                              (Intrinsic::ID)id));
153  return Upgraded;
154}
155
156bool llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
157  // Nothing to do yet.
158  return false;
159}
160
161// UpgradeIntrinsicCall - Upgrade a call to an old intrinsic to be a call the
162// upgraded intrinsic. All argument and return casting must be provided in
163// order to seamlessly integrate with existing context.
164void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
165  Function *F = CI->getCalledFunction();
166  LLVMContext &C = CI->getContext();
167  IRBuilder<> Builder(C);
168  Builder.SetInsertPoint(CI->getParent(), CI);
169
170  assert(F && "Intrinsic call is not direct?");
171
172  if (!NewFn) {
173    // Get the Function's name.
174    StringRef Name = F->getName();
175
176    Value *Rep;
177    // Upgrade packed integer vector compares intrinsics to compare instructions
178    if (Name.startswith("llvm.x86.sse2.pcmpeq.") ||
179        Name.startswith("llvm.x86.avx2.pcmpeq.")) {
180      Rep = Builder.CreateICmpEQ(CI->getArgOperand(0), CI->getArgOperand(1),
181                                 "pcmpeq");
182      // need to sign extend since icmp returns vector of i1
183      Rep = Builder.CreateSExt(Rep, CI->getType(), "");
184    } else if (Name.startswith("llvm.x86.sse2.pcmpgt.") ||
185               Name.startswith("llvm.x86.avx2.pcmpgt.")) {
186      Rep = Builder.CreateICmpSGT(CI->getArgOperand(0), CI->getArgOperand(1),
187                                  "pcmpgt");
188      // need to sign extend since icmp returns vector of i1
189      Rep = Builder.CreateSExt(Rep, CI->getType(), "");
190    } else if (Name == "llvm.x86.avx.movnt.dq.256" ||
191               Name == "llvm.x86.avx.movnt.ps.256" ||
192               Name == "llvm.x86.avx.movnt.pd.256") {
193      IRBuilder<> Builder(C);
194      Builder.SetInsertPoint(CI->getParent(), CI);
195
196      Module *M = F->getParent();
197      SmallVector<Value *, 1> Elts;
198      Elts.push_back(ConstantInt::get(Type::getInt32Ty(C), 1));
199      MDNode *Node = MDNode::get(C, Elts);
200
201      Value *Arg0 = CI->getArgOperand(0);
202      Value *Arg1 = CI->getArgOperand(1);
203
204      // Convert the type of the pointer to a pointer to the stored type.
205      Value *BC = Builder.CreateBitCast(Arg0,
206                                        PointerType::getUnqual(Arg1->getType()),
207                                        "cast");
208      StoreInst *SI = Builder.CreateStore(Arg1, BC);
209      SI->setMetadata(M->getMDKindID("nontemporal"), Node);
210      SI->setAlignment(16);
211
212      // Remove intrinsic.
213      CI->eraseFromParent();
214      return;
215    } else if (Name.startswith("llvm.x86.xop.vpcom")) {
216      Intrinsic::ID intID;
217      if (Name.endswith("ub"))
218        intID = Intrinsic::x86_xop_vpcomub;
219      else if (Name.endswith("uw"))
220        intID = Intrinsic::x86_xop_vpcomuw;
221      else if (Name.endswith("ud"))
222        intID = Intrinsic::x86_xop_vpcomud;
223      else if (Name.endswith("uq"))
224        intID = Intrinsic::x86_xop_vpcomuq;
225      else if (Name.endswith("b"))
226        intID = Intrinsic::x86_xop_vpcomb;
227      else if (Name.endswith("w"))
228        intID = Intrinsic::x86_xop_vpcomw;
229      else if (Name.endswith("d"))
230        intID = Intrinsic::x86_xop_vpcomd;
231      else if (Name.endswith("q"))
232        intID = Intrinsic::x86_xop_vpcomq;
233      else
234        llvm_unreachable("Unknown suffix");
235
236      Name = Name.substr(18); // strip off "llvm.x86.xop.vpcom"
237      unsigned Imm;
238      if (Name.startswith("lt"))
239        Imm = 0;
240      else if (Name.startswith("le"))
241        Imm = 1;
242      else if (Name.startswith("gt"))
243        Imm = 2;
244      else if (Name.startswith("ge"))
245        Imm = 3;
246      else if (Name.startswith("eq"))
247        Imm = 4;
248      else if (Name.startswith("ne"))
249        Imm = 5;
250      else if (Name.startswith("true"))
251        Imm = 6;
252      else if (Name.startswith("false"))
253        Imm = 7;
254      else
255        llvm_unreachable("Unknown condition");
256
257      Function *VPCOM = Intrinsic::getDeclaration(F->getParent(), intID);
258      Rep = Builder.CreateCall3(VPCOM, CI->getArgOperand(0),
259                                CI->getArgOperand(1), Builder.getInt8(Imm));
260    } else {
261      bool PD128 = false, PD256 = false, PS128 = false, PS256 = false;
262      if (Name == "llvm.x86.avx.vpermil.pd.256")
263        PD256 = true;
264      else if (Name == "llvm.x86.avx.vpermil.pd")
265        PD128 = true;
266      else if (Name == "llvm.x86.avx.vpermil.ps.256")
267        PS256 = true;
268      else if (Name == "llvm.x86.avx.vpermil.ps")
269        PS128 = true;
270
271      if (PD256 || PD128 || PS256 || PS128) {
272        Value *Op0 = CI->getArgOperand(0);
273        unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
274        SmallVector<Constant*, 8> Idxs;
275
276        if (PD128)
277          for (unsigned i = 0; i != 2; ++i)
278            Idxs.push_back(Builder.getInt32((Imm >> i) & 0x1));
279        else if (PD256)
280          for (unsigned l = 0; l != 4; l+=2)
281            for (unsigned i = 0; i != 2; ++i)
282              Idxs.push_back(Builder.getInt32(((Imm >> (l+i)) & 0x1) + l));
283        else if (PS128)
284          for (unsigned i = 0; i != 4; ++i)
285            Idxs.push_back(Builder.getInt32((Imm >> (2 * i)) & 0x3));
286        else if (PS256)
287          for (unsigned l = 0; l != 8; l+=4)
288            for (unsigned i = 0; i != 4; ++i)
289              Idxs.push_back(Builder.getInt32(((Imm >> (2 * i)) & 0x3) + l));
290        else
291          llvm_unreachable("Unexpected function");
292
293        Rep = Builder.CreateShuffleVector(Op0, Op0, ConstantVector::get(Idxs));
294      } else {
295        llvm_unreachable("Unknown function for CallInst upgrade.");
296      }
297    }
298
299    CI->replaceAllUsesWith(Rep);
300    CI->eraseFromParent();
301    return;
302  }
303
304  std::string Name = CI->getName().str();
305  CI->setName(Name + ".old");
306
307  switch (NewFn->getIntrinsicID()) {
308  default:
309    llvm_unreachable("Unknown function for CallInst upgrade.");
310
311  case Intrinsic::ctlz:
312  case Intrinsic::cttz:
313    assert(CI->getNumArgOperands() == 1 &&
314           "Mismatch between function args and call args");
315    CI->replaceAllUsesWith(Builder.CreateCall2(NewFn, CI->getArgOperand(0),
316                                               Builder.getFalse(), Name));
317    CI->eraseFromParent();
318    return;
319
320  case Intrinsic::arm_neon_vclz: {
321    // Change name from llvm.arm.neon.vclz.* to llvm.ctlz.*
322    CI->replaceAllUsesWith(Builder.CreateCall2(NewFn, CI->getArgOperand(0),
323                                               Builder.getFalse(),
324                                               "llvm.ctlz." + Name.substr(14)));
325    CI->eraseFromParent();
326    return;
327  }
328  case Intrinsic::ctpop: {
329    CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(0)));
330    CI->eraseFromParent();
331    return;
332  }
333
334  case Intrinsic::x86_xop_vfrcz_ss:
335  case Intrinsic::x86_xop_vfrcz_sd:
336    CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(1),
337                                              Name));
338    CI->eraseFromParent();
339    return;
340
341  case Intrinsic::x86_sse41_ptestc:
342  case Intrinsic::x86_sse41_ptestz:
343  case Intrinsic::x86_sse41_ptestnzc: {
344    // The arguments for these intrinsics used to be v4f32, and changed
345    // to v2i64. This is purely a nop, since those are bitwise intrinsics.
346    // So, the only thing required is a bitcast for both arguments.
347    // First, check the arguments have the old type.
348    Value *Arg0 = CI->getArgOperand(0);
349    if (Arg0->getType() != VectorType::get(Type::getFloatTy(C), 4))
350      return;
351
352    // Old intrinsic, add bitcasts
353    Value *Arg1 = CI->getArgOperand(1);
354
355    Value *BC0 =
356      Builder.CreateBitCast(Arg0,
357                            VectorType::get(Type::getInt64Ty(C), 2),
358                            "cast");
359    Value *BC1 =
360      Builder.CreateBitCast(Arg1,
361                            VectorType::get(Type::getInt64Ty(C), 2),
362                            "cast");
363
364    CallInst* NewCall = Builder.CreateCall2(NewFn, BC0, BC1, Name);
365    CI->replaceAllUsesWith(NewCall);
366    CI->eraseFromParent();
367    return;
368  }
369  }
370}
371
372// This tests each Function to determine if it needs upgrading. When we find
373// one we are interested in, we then upgrade all calls to reflect the new
374// function.
375void llvm::UpgradeCallsToIntrinsic(Function* F) {
376  assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
377
378  // Upgrade the function and check if it is a totaly new function.
379  Function *NewFn;
380  if (UpgradeIntrinsicFunction(F, NewFn)) {
381    if (NewFn != F) {
382      // Replace all uses to the old function with the new one if necessary.
383      for (Value::use_iterator UI = F->use_begin(), UE = F->use_end();
384           UI != UE; ) {
385        if (CallInst *CI = dyn_cast<CallInst>(*UI++))
386          UpgradeIntrinsicCall(CI, NewFn);
387      }
388      // Remove old function, no longer used, from the module.
389      F->eraseFromParent();
390    }
391  }
392}
393
394void llvm::UpgradeInstWithTBAATag(Instruction *I) {
395  MDNode *MD = I->getMetadata(LLVMContext::MD_tbaa);
396  assert(MD && "UpgradeInstWithTBAATag should have a TBAA tag");
397  // Check if the tag uses struct-path aware TBAA format.
398  if (isa<MDNode>(MD->getOperand(0)) && MD->getNumOperands() >= 3)
399    return;
400
401  if (MD->getNumOperands() == 3) {
402    Value *Elts[] = {
403      MD->getOperand(0),
404      MD->getOperand(1)
405    };
406    MDNode *ScalarType = MDNode::get(I->getContext(), Elts);
407    // Create a MDNode <ScalarType, ScalarType, offset 0, const>
408    Value *Elts2[] = {
409      ScalarType, ScalarType,
410      Constant::getNullValue(Type::getInt64Ty(I->getContext())),
411      MD->getOperand(2)
412    };
413    I->setMetadata(LLVMContext::MD_tbaa, MDNode::get(I->getContext(), Elts2));
414  } else {
415    // Create a MDNode <MD, MD, offset 0>
416    Value *Elts[] = {MD, MD,
417      Constant::getNullValue(Type::getInt64Ty(I->getContext()))};
418    I->setMetadata(LLVMContext::MD_tbaa, MDNode::get(I->getContext(), Elts));
419  }
420}
421