1//===- LowerExpectIntrinsic.cpp - Lower expect intrinsic ------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This pass lowers the 'expect' intrinsic to LLVM metadata.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Scalar.h"
15#include "llvm/ADT/Statistic.h"
16#include "llvm/IR/BasicBlock.h"
17#include "llvm/IR/Constants.h"
18#include "llvm/IR/Function.h"
19#include "llvm/IR/Instructions.h"
20#include "llvm/IR/Intrinsics.h"
21#include "llvm/IR/LLVMContext.h"
22#include "llvm/IR/MDBuilder.h"
23#include "llvm/IR/Metadata.h"
24#include "llvm/Pass.h"
25#include "llvm/Support/CommandLine.h"
26#include "llvm/Support/Debug.h"
27#include <vector>
28
29using namespace llvm;
30
31#define DEBUG_TYPE "lower-expect-intrinsic"
32
33STATISTIC(IfHandled, "Number of 'expect' intrinsic instructions handled");
34
35static cl::opt<uint32_t>
36LikelyBranchWeight("likely-branch-weight", cl::Hidden, cl::init(64),
37                   cl::desc("Weight of the branch likely to be taken (default = 64)"));
38static cl::opt<uint32_t>
39UnlikelyBranchWeight("unlikely-branch-weight", cl::Hidden, cl::init(4),
40                   cl::desc("Weight of the branch unlikely to be taken (default = 4)"));
41
42namespace {
43
44  class LowerExpectIntrinsic : public FunctionPass {
45
46    bool HandleSwitchExpect(SwitchInst *SI);
47
48    bool HandleIfExpect(BranchInst *BI);
49
50  public:
51    static char ID;
52    LowerExpectIntrinsic() : FunctionPass(ID) {
53      initializeLowerExpectIntrinsicPass(*PassRegistry::getPassRegistry());
54    }
55
56    bool runOnFunction(Function &F) override;
57  };
58}
59
60
61bool LowerExpectIntrinsic::HandleSwitchExpect(SwitchInst *SI) {
62  CallInst *CI = dyn_cast<CallInst>(SI->getCondition());
63  if (!CI)
64    return false;
65
66  Function *Fn = CI->getCalledFunction();
67  if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect)
68    return false;
69
70  Value *ArgValue = CI->getArgOperand(0);
71  ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1));
72  if (!ExpectedValue)
73    return false;
74
75  SwitchInst::CaseIt Case = SI->findCaseValue(ExpectedValue);
76  unsigned n = SI->getNumCases(); // +1 for default case.
77  std::vector<uint32_t> Weights(n + 1);
78
79  Weights[0] = Case == SI->case_default() ? LikelyBranchWeight
80                                          : UnlikelyBranchWeight;
81  for (unsigned i = 0; i != n; ++i)
82    Weights[i + 1] = i == Case.getCaseIndex() ? LikelyBranchWeight
83                                              : UnlikelyBranchWeight;
84
85  SI->setMetadata(LLVMContext::MD_prof,
86                  MDBuilder(CI->getContext()).createBranchWeights(Weights));
87
88  SI->setCondition(ArgValue);
89  return true;
90}
91
92
93bool LowerExpectIntrinsic::HandleIfExpect(BranchInst *BI) {
94  if (BI->isUnconditional())
95    return false;
96
97  // Handle non-optimized IR code like:
98  //   %expval = call i64 @llvm.expect.i64(i64 %conv1, i64 1)
99  //   %tobool = icmp ne i64 %expval, 0
100  //   br i1 %tobool, label %if.then, label %if.end
101  //
102  // Or the following simpler case:
103  //   %expval = call i1 @llvm.expect.i1(i1 %cmp, i1 1)
104  //   br i1 %expval, label %if.then, label %if.end
105
106  CallInst *CI;
107
108  ICmpInst *CmpI = dyn_cast<ICmpInst>(BI->getCondition());
109  if (!CmpI) {
110    CI = dyn_cast<CallInst>(BI->getCondition());
111  } else {
112    if (CmpI->getPredicate() != CmpInst::ICMP_NE)
113      return false;
114    CI = dyn_cast<CallInst>(CmpI->getOperand(0));
115  }
116
117  if (!CI)
118    return false;
119
120  Function *Fn = CI->getCalledFunction();
121  if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect)
122    return false;
123
124  Value *ArgValue = CI->getArgOperand(0);
125  ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1));
126  if (!ExpectedValue)
127    return false;
128
129  MDBuilder MDB(CI->getContext());
130  MDNode *Node;
131
132  // If expect value is equal to 1 it means that we are more likely to take
133  // branch 0, in other case more likely is branch 1.
134  if (ExpectedValue->isOne())
135    Node = MDB.createBranchWeights(LikelyBranchWeight, UnlikelyBranchWeight);
136  else
137    Node = MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight);
138
139  BI->setMetadata(LLVMContext::MD_prof, Node);
140
141  if (CmpI)
142    CmpI->setOperand(0, ArgValue);
143  else
144    BI->setCondition(ArgValue);
145  return true;
146}
147
148
149bool LowerExpectIntrinsic::runOnFunction(Function &F) {
150  for (Function::iterator I = F.begin(), E = F.end(); I != E;) {
151    BasicBlock *BB = I++;
152
153    // Create "block_weights" metadata.
154    if (BranchInst *BI = dyn_cast<BranchInst>(BB->getTerminator())) {
155      if (HandleIfExpect(BI))
156        IfHandled++;
157    } else if (SwitchInst *SI = dyn_cast<SwitchInst>(BB->getTerminator())) {
158      if (HandleSwitchExpect(SI))
159        IfHandled++;
160    }
161
162    // remove llvm.expect intrinsics.
163    for (BasicBlock::iterator BI = BB->begin(), BE = BB->end();
164         BI != BE; ) {
165      CallInst *CI = dyn_cast<CallInst>(BI++);
166      if (!CI)
167        continue;
168
169      Function *Fn = CI->getCalledFunction();
170      if (Fn && Fn->getIntrinsicID() == Intrinsic::expect) {
171        Value *Exp = CI->getArgOperand(0);
172        CI->replaceAllUsesWith(Exp);
173        CI->eraseFromParent();
174      }
175    }
176  }
177
178  return false;
179}
180
181
182char LowerExpectIntrinsic::ID = 0;
183INITIALIZE_PASS(LowerExpectIntrinsic, "lower-expect", "Lower 'expect' "
184                "Intrinsics", false, false)
185
186FunctionPass *llvm::createLowerExpectIntrinsicPass() {
187  return new LowerExpectIntrinsic();
188}
189