1//===- LowerAtomic.cpp - Lower atomic intrinsics --------------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This pass lowers atomic intrinsics to non-atomic form for use in a known
11// non-preemptible environment.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/Transforms/Scalar.h"
16#include "llvm/IR/Function.h"
17#include "llvm/IR/IRBuilder.h"
18#include "llvm/IR/IntrinsicInst.h"
19#include "llvm/Pass.h"
20using namespace llvm;
21
22#define DEBUG_TYPE "loweratomic"
23
24static bool LowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) {
25  IRBuilder<> Builder(CXI->getParent(), CXI);
26  Value *Ptr = CXI->getPointerOperand();
27  Value *Cmp = CXI->getCompareOperand();
28  Value *Val = CXI->getNewValOperand();
29
30  LoadInst *Orig = Builder.CreateLoad(Ptr);
31  Value *Equal = Builder.CreateICmpEQ(Orig, Cmp);
32  Value *Res = Builder.CreateSelect(Equal, Val, Orig);
33  Builder.CreateStore(Res, Ptr);
34
35  Res = Builder.CreateInsertValue(UndefValue::get(CXI->getType()), Orig, 0);
36  Res = Builder.CreateInsertValue(Res, Equal, 1);
37
38  CXI->replaceAllUsesWith(Res);
39  CXI->eraseFromParent();
40  return true;
41}
42
43static bool LowerAtomicRMWInst(AtomicRMWInst *RMWI) {
44  IRBuilder<> Builder(RMWI->getParent(), RMWI);
45  Value *Ptr = RMWI->getPointerOperand();
46  Value *Val = RMWI->getValOperand();
47
48  LoadInst *Orig = Builder.CreateLoad(Ptr);
49  Value *Res = nullptr;
50
51  switch (RMWI->getOperation()) {
52  default: llvm_unreachable("Unexpected RMW operation");
53  case AtomicRMWInst::Xchg:
54    Res = Val;
55    break;
56  case AtomicRMWInst::Add:
57    Res = Builder.CreateAdd(Orig, Val);
58    break;
59  case AtomicRMWInst::Sub:
60    Res = Builder.CreateSub(Orig, Val);
61    break;
62  case AtomicRMWInst::And:
63    Res = Builder.CreateAnd(Orig, Val);
64    break;
65  case AtomicRMWInst::Nand:
66    Res = Builder.CreateNot(Builder.CreateAnd(Orig, Val));
67    break;
68  case AtomicRMWInst::Or:
69    Res = Builder.CreateOr(Orig, Val);
70    break;
71  case AtomicRMWInst::Xor:
72    Res = Builder.CreateXor(Orig, Val);
73    break;
74  case AtomicRMWInst::Max:
75    Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Val),
76                               Val, Orig);
77    break;
78  case AtomicRMWInst::Min:
79    Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Val),
80                               Orig, Val);
81    break;
82  case AtomicRMWInst::UMax:
83    Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Val),
84                               Val, Orig);
85    break;
86  case AtomicRMWInst::UMin:
87    Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Val),
88                               Orig, Val);
89    break;
90  }
91  Builder.CreateStore(Res, Ptr);
92  RMWI->replaceAllUsesWith(Orig);
93  RMWI->eraseFromParent();
94  return true;
95}
96
97static bool LowerFenceInst(FenceInst *FI) {
98  FI->eraseFromParent();
99  return true;
100}
101
102static bool LowerLoadInst(LoadInst *LI) {
103  LI->setAtomic(NotAtomic);
104  return true;
105}
106
107static bool LowerStoreInst(StoreInst *SI) {
108  SI->setAtomic(NotAtomic);
109  return true;
110}
111
112namespace {
113  struct LowerAtomic : public BasicBlockPass {
114    static char ID;
115    LowerAtomic() : BasicBlockPass(ID) {
116      initializeLowerAtomicPass(*PassRegistry::getPassRegistry());
117    }
118    bool runOnBasicBlock(BasicBlock &BB) override {
119      if (skipOptnoneFunction(BB))
120        return false;
121      bool Changed = false;
122      for (BasicBlock::iterator DI = BB.begin(), DE = BB.end(); DI != DE; ) {
123        Instruction *Inst = DI++;
124        if (FenceInst *FI = dyn_cast<FenceInst>(Inst))
125          Changed |= LowerFenceInst(FI);
126        else if (AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(Inst))
127          Changed |= LowerAtomicCmpXchgInst(CXI);
128        else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(Inst))
129          Changed |= LowerAtomicRMWInst(RMWI);
130        else if (LoadInst *LI = dyn_cast<LoadInst>(Inst)) {
131          if (LI->isAtomic())
132            LowerLoadInst(LI);
133        } else if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
134          if (SI->isAtomic())
135            LowerStoreInst(SI);
136        }
137      }
138      return Changed;
139    }
140  };
141}
142
143char LowerAtomic::ID = 0;
144INITIALIZE_PASS(LowerAtomic, "loweratomic",
145                "Lower atomic intrinsics to non-atomic form",
146                false, false)
147
148Pass *llvm::createLowerAtomicPass() { return new LowerAtomic(); }
149