LowerAtomic.cpp revision 21006d40ac9ec7715bca2095451075a83773dc52 
1//===- LowerAtomic.cpp - Lower atomic intrinsics --------------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This pass lowers atomic intrinsics to non-atomic form for use in a known
11// non-preemptible environment.
12//
13//===----------------------------------------------------------------------===//
14
15#define DEBUG_TYPE "loweratomic"
16#include "llvm/Transforms/Scalar.h"
17#include "llvm/Function.h"
18#include "llvm/IntrinsicInst.h"
19#include "llvm/Pass.h"
20#include "llvm/Support/IRBuilder.h"
21using namespace llvm;
22
23static bool LowerAtomicIntrinsic(IntrinsicInst *II) {
24  IRBuilder<> Builder(II->getParent(), II);
25  unsigned IID = II->getIntrinsicID();
26  switch (IID) {
27  case Intrinsic::memory_barrier:
28    break;
29
30  case Intrinsic::atomic_load_add:
31  case Intrinsic::atomic_load_sub:
32  case Intrinsic::atomic_load_and:
33  case Intrinsic::atomic_load_nand:
34  case Intrinsic::atomic_load_or:
35  case Intrinsic::atomic_load_xor:
36  case Intrinsic::atomic_load_max:
37  case Intrinsic::atomic_load_min:
38  case Intrinsic::atomic_load_umax:
39  case Intrinsic::atomic_load_umin: {
40    Value *Ptr = II->getArgOperand(0), *Delta = II->getArgOperand(1);
41
42    LoadInst *Orig = Builder.CreateLoad(Ptr);
43    Value *Res = NULL;
44    switch (IID) {
45    default: assert(0 && "Unrecognized atomic modify operation");
46    case Intrinsic::atomic_load_add:
47      Res = Builder.CreateAdd(Orig, Delta);
48      break;
49    case Intrinsic::atomic_load_sub:
50      Res = Builder.CreateSub(Orig, Delta);
51      break;
52    case Intrinsic::atomic_load_and:
53      Res = Builder.CreateAnd(Orig, Delta);
54      break;
55    case Intrinsic::atomic_load_nand:
56      Res = Builder.CreateNot(Builder.CreateAnd(Orig, Delta));
57      break;
58    case Intrinsic::atomic_load_or:
59      Res = Builder.CreateOr(Orig, Delta);
60      break;
61    case Intrinsic::atomic_load_xor:
62      Res = Builder.CreateXor(Orig, Delta);
63      break;
64    case Intrinsic::atomic_load_max:
65      Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Delta),
66                                 Delta, Orig);
67      break;
68    case Intrinsic::atomic_load_min:
69      Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Delta),
70                                 Orig, Delta);
71      break;
72    case Intrinsic::atomic_load_umax:
73      Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Delta),
74                                 Delta, Orig);
75      break;
76    case Intrinsic::atomic_load_umin:
77      Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Delta),
78                                 Orig, Delta);
79      break;
80    }
81    Builder.CreateStore(Res, Ptr);
82
83    II->replaceAllUsesWith(Orig);
84    break;
85  }
86
87  case Intrinsic::atomic_swap: {
88    Value *Ptr = II->getArgOperand(0), *Val = II->getArgOperand(1);
89    LoadInst *Orig = Builder.CreateLoad(Ptr);
90    Builder.CreateStore(Val, Ptr);
91    II->replaceAllUsesWith(Orig);
92    break;
93  }
94
95  case Intrinsic::atomic_cmp_swap: {
96    Value *Ptr = II->getArgOperand(0), *Cmp = II->getArgOperand(1);
97    Value *Val = II->getArgOperand(2);
98
99    LoadInst *Orig = Builder.CreateLoad(Ptr);
100    Value *Equal = Builder.CreateICmpEQ(Orig, Cmp);
101    Value *Res = Builder.CreateSelect(Equal, Val, Orig);
102    Builder.CreateStore(Res, Ptr);
103    II->replaceAllUsesWith(Orig);
104    break;
105  }
106
107  default:
108    return false;
109  }
110
111  assert(II->use_empty() &&
112         "Lowering should have eliminated any uses of the intrinsic call!");
113  II->eraseFromParent();
114
115  return true;
116}
117
118bool LowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) {
119  IRBuilder<> Builder(CXI->getParent(), CXI);
120  Value *Ptr = CXI->getPointerOperand();
121  Value *Cmp = CXI->getCompareOperand();
122  Value *Val = CXI->getNewValOperand();
123
124  LoadInst *Orig = Builder.CreateLoad(Ptr);
125  Value *Equal = Builder.CreateICmpEQ(Orig, Cmp);
126  Value *Res = Builder.CreateSelect(Equal, Val, Orig);
127  Builder.CreateStore(Res, Ptr);
128
129  CXI->replaceAllUsesWith(Orig);
130  CXI->eraseFromParent();
131  return true;
132}
133
134bool LowerAtomicRMWInst(AtomicRMWInst *RMWI) {
135  IRBuilder<> Builder(RMWI->getParent(), RMWI);
136  Value *Ptr = RMWI->getPointerOperand();
137  Value *Val = RMWI->getValOperand();
138
139  LoadInst *Orig = Builder.CreateLoad(Ptr);
140  Value *Res = NULL;
141
142  switch (RMWI->getOperation()) {
143  default: llvm_unreachable("Unexpected RMW operation");
144  case AtomicRMWInst::Xchg:
145    Res = Val;
146    break;
147  case AtomicRMWInst::Add:
148    Res = Builder.CreateAdd(Orig, Val);
149    break;
150  case AtomicRMWInst::Sub:
151    Res = Builder.CreateSub(Orig, Val);
152    break;
153  case AtomicRMWInst::And:
154    Res = Builder.CreateAnd(Orig, Val);
155    break;
156  case AtomicRMWInst::Nand:
157    Res = Builder.CreateNot(Builder.CreateAnd(Orig, Val));
158    break;
159  case AtomicRMWInst::Or:
160    Res = Builder.CreateOr(Orig, Val);
161    break;
162  case AtomicRMWInst::Xor:
163    Res = Builder.CreateXor(Orig, Val);
164    break;
165  case AtomicRMWInst::Max:
166    Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Val),
167                               Val, Orig);
168    break;
169  case AtomicRMWInst::Min:
170    Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Val),
171                               Orig, Val);
172    break;
173  case AtomicRMWInst::UMax:
174    Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Val),
175                               Val, Orig);
176    break;
177  case AtomicRMWInst::UMin:
178    Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Val),
179                               Orig, Val);
180    break;
181  }
182  Builder.CreateStore(Res, Ptr);
183  RMWI->replaceAllUsesWith(Orig);
184  RMWI->eraseFromParent();
185  return true;
186}
187
188static bool LowerFenceInst(FenceInst *FI) {
189  FI->eraseFromParent();
190  return true;
191}
192
193static bool LowerLoadInst(LoadInst *LI) {
194  LI->setAtomic(NotAtomic);
195  return true;
196}
197
198static bool LowerStoreInst(StoreInst *SI) {
199  SI->setAtomic(NotAtomic);
200  return true;
201}
202
203namespace {
204  struct LowerAtomic : public BasicBlockPass {
205    static char ID;
206    LowerAtomic() : BasicBlockPass(ID) {
207      initializeLowerAtomicPass(*PassRegistry::getPassRegistry());
208    }
209    bool runOnBasicBlock(BasicBlock &BB) {
210      bool Changed = false;
211      for (BasicBlock::iterator DI = BB.begin(), DE = BB.end(); DI != DE; ) {
212        Instruction *Inst = DI++;
213        if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst))
214          Changed |= LowerAtomicIntrinsic(II);
215        else if (FenceInst *FI = dyn_cast<FenceInst>(Inst))
216          Changed |= LowerFenceInst(FI);
217        else if (AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(Inst))
218          Changed |= LowerAtomicCmpXchgInst(CXI);
219        else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(Inst))
220          Changed |= LowerAtomicRMWInst(RMWI);
221        else if (LoadInst *LI = dyn_cast<LoadInst>(Inst)) {
222          if (LI->isAtomic())
223            LowerLoadInst(LI);
224        } else if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
225          if (SI->isAtomic())
226            LowerStoreInst(SI);
227        }
228      }
229      return Changed;
230    }
231  };
232}
233
234char LowerAtomic::ID = 0;
235INITIALIZE_PASS(LowerAtomic, "loweratomic",
236                "Lower atomic intrinsics to non-atomic form",
237                false, false)
238
239Pass *llvm::createLowerAtomicPass() { return new LowerAtomic(); }
240