AtomicExpandPass.cpp revision f3ef5332fa3f4d5ec72c178a2b19dac363a19383
137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines//===-- AtomicExpandPass.cpp - Expand atomic instructions -------===// 237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// 337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// The LLVM Compiler Infrastructure 437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// 537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// This file is distributed under the University of Illinois Open Source 637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// License. See LICENSE.TXT for details. 737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// 837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines//===----------------------------------------------------------------------===// 937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// 1037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// This file contains a pass (at IR level) to replace atomic instructions with 11f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar// target specific instruction which implement the same semantics in a way 12f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar// which better fits the target backend. This can include the use of either 13f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar// (intrinsic-based) load-linked/store-conditional loops, AtomicCmpXchg, or 14f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar// type coercions. 1537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// 1637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines//===----------------------------------------------------------------------===// 1737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 18f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar#include "llvm/CodeGen/AtomicExpandUtils.h" 1937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "llvm/CodeGen/Passes.h" 2037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "llvm/IR/Function.h" 2137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "llvm/IR/IRBuilder.h" 2237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "llvm/IR/InstIterator.h" 2337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "llvm/IR/Instructions.h" 2437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "llvm/IR/Intrinsics.h" 2537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "llvm/IR/Module.h" 2637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "llvm/Support/Debug.h" 27f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar#include "llvm/Support/raw_ostream.h" 2837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "llvm/Target/TargetLowering.h" 2937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "llvm/Target/TargetMachine.h" 3037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "llvm/Target/TargetSubtargetInfo.h" 3137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 3237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesusing namespace llvm; 3337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 3437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#define DEBUG_TYPE "atomic-expand" 3537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 3637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesnamespace { 3737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines class AtomicExpand: public FunctionPass { 3837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines const TargetMachine *TM; 39ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const TargetLowering *TLI; 4037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines public: 4137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines static char ID; // Pass identification, replacement for typeid 4237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines explicit AtomicExpand(const TargetMachine *TM = nullptr) 43ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines : FunctionPass(ID), TM(TM), TLI(nullptr) { 4437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines initializeAtomicExpandPass(*PassRegistry::getPassRegistry()); 4537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 4637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 4737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool runOnFunction(Function &F) override; 4837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 4937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines private: 5037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool bracketInstWithFences(Instruction *I, AtomicOrdering Order, 5137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool IsStore, bool IsLoad); 52f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar IntegerType *getCorrespondingIntegerType(Type *T, const DataLayout &DL); 53f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar LoadInst *convertAtomicLoadToIntegerType(LoadInst *LI); 54f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar bool tryExpandAtomicLoad(LoadInst *LI); 5537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool expandAtomicLoadToLL(LoadInst *LI); 5637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool expandAtomicLoadToCmpXchg(LoadInst *LI); 57f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar StoreInst *convertAtomicStoreToIntegerType(StoreInst *SI); 5837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool expandAtomicStore(StoreInst *SI); 594c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar bool tryExpandAtomicRMW(AtomicRMWInst *AI); 60f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar bool expandAtomicOpToLLSC( 61f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Instruction *I, Value *Addr, AtomicOrdering MemOpOrder, 62f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar std::function<Value *(IRBuilder<> &, Value *)> PerformOp); 6337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool expandAtomicCmpXchg(AtomicCmpXchgInst *CI); 6437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool isIdempotentRMW(AtomicRMWInst *AI); 6537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool simplifyIdempotentRMW(AtomicRMWInst *AI); 6637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines }; 6737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 6837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 6937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hineschar AtomicExpand::ID = 0; 7037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hineschar &llvm::AtomicExpandID = AtomicExpand::ID; 7137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen HinesINITIALIZE_TM_PASS(AtomicExpand, "atomic-expand", 7237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines "Expand Atomic calls in terms of either load-linked & store-conditional or cmpxchg", 7337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines false, false) 7437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 7537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen HinesFunctionPass *llvm::createAtomicExpandPass(const TargetMachine *TM) { 7637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return new AtomicExpand(TM); 7737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 7837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 7937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesbool AtomicExpand::runOnFunction(Function &F) { 80ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!TM || !TM->getSubtargetImpl(F)->enableAtomicExpand()) 8137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 82ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines TLI = TM->getSubtargetImpl(F)->getTargetLowering(); 8337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 8437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SmallVector<Instruction *, 1> AtomicInsts; 8537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 8637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Changing control-flow while iterating through it is a bad idea, so gather a 8737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // list of all atomic instructions before we start. 8837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines for (inst_iterator I = inst_begin(F), E = inst_end(F); I != E; ++I) { 8937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (I->isAtomic()) 9037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AtomicInsts.push_back(&*I); 9137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 9237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 9337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool MadeChange = false; 9437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines for (auto I : AtomicInsts) { 9537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines auto LI = dyn_cast<LoadInst>(I); 9637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines auto SI = dyn_cast<StoreInst>(I); 9737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines auto RMWI = dyn_cast<AtomicRMWInst>(I); 9837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines auto CASI = dyn_cast<AtomicCmpXchgInst>(I); 9937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines assert((LI || SI || RMWI || CASI || isa<FenceInst>(I)) && 10037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines "Unknown atomic instruction"); 10137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 10237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines auto FenceOrdering = Monotonic; 10337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool IsStore, IsLoad; 104ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (TLI->getInsertFencesForAtomic()) { 10537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (LI && isAtLeastAcquire(LI->getOrdering())) { 10637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines FenceOrdering = LI->getOrdering(); 10737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines LI->setOrdering(Monotonic); 10837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IsStore = false; 10937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IsLoad = true; 11037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } else if (SI && isAtLeastRelease(SI->getOrdering())) { 11137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines FenceOrdering = SI->getOrdering(); 11237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SI->setOrdering(Monotonic); 11337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IsStore = true; 11437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IsLoad = false; 11537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } else if (RMWI && (isAtLeastRelease(RMWI->getOrdering()) || 11637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines isAtLeastAcquire(RMWI->getOrdering()))) { 11737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines FenceOrdering = RMWI->getOrdering(); 11837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines RMWI->setOrdering(Monotonic); 11937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IsStore = IsLoad = true; 120f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } else if (CASI && !TLI->shouldExpandAtomicCmpXchgInIR(CASI) && 121ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines (isAtLeastRelease(CASI->getSuccessOrdering()) || 122ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines isAtLeastAcquire(CASI->getSuccessOrdering()))) { 12337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // If a compare and swap is lowered to LL/SC, we can do smarter fence 12437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // insertion, with a stronger one on the success path than on the 12537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // failure path. As a result, fence insertion is directly done by 12637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // expandAtomicCmpXchg in that case. 12737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines FenceOrdering = CASI->getSuccessOrdering(); 12837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CASI->setSuccessOrdering(Monotonic); 12937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CASI->setFailureOrdering(Monotonic); 13037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IsStore = IsLoad = true; 13137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 13237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 13337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (FenceOrdering != Monotonic) { 13437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines MadeChange |= bracketInstWithFences(I, FenceOrdering, IsStore, IsLoad); 13537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 13637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 13737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 138f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (LI) { 139f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (LI->getType()->isFloatingPointTy()) { 140f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // TODO: add a TLI hook to control this so that each target can 141f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // convert to lowering the original type one at a time. 142f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar LI = convertAtomicLoadToIntegerType(LI); 143f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar assert(LI->getType()->isIntegerTy() && "invariant broken"); 144f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MadeChange = true; 145f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 146f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 147f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MadeChange |= tryExpandAtomicLoad(LI); 148f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } else if (SI) { 149f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (SI->getValueOperand()->getType()->isFloatingPointTy()) { 150f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // TODO: add a TLI hook to control this so that each target can 151f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // convert to lowering the original type one at a time. 152f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SI = convertAtomicStoreToIntegerType(SI); 153f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar assert(SI->getValueOperand()->getType()->isIntegerTy() && 154f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar "invariant broken"); 155f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MadeChange = true; 156f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 157f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 158f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (TLI->shouldExpandAtomicStoreInIR(SI)) 159f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MadeChange |= expandAtomicStore(SI); 16037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } else if (RMWI) { 16137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // There are two different ways of expanding RMW instructions: 16237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // - into a load if it is idempotent 16337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // - into a Cmpxchg/LL-SC loop otherwise 16437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // we try them in that order. 1654c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar 1664c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar if (isIdempotentRMW(RMWI) && simplifyIdempotentRMW(RMWI)) { 1674c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar MadeChange = true; 1684c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar } else { 1694c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar MadeChange |= tryExpandAtomicRMW(RMWI); 1704c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar } 171f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } else if (CASI && TLI->shouldExpandAtomicCmpXchgInIR(CASI)) { 17237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines MadeChange |= expandAtomicCmpXchg(CASI); 17337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 17437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 17537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return MadeChange; 17637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 17737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 17837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesbool AtomicExpand::bracketInstWithFences(Instruction *I, AtomicOrdering Order, 17937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool IsStore, bool IsLoad) { 18037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IRBuilder<> Builder(I); 18137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 182ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines auto LeadingFence = TLI->emitLeadingFence(Builder, Order, IsStore, IsLoad); 18337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 184ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines auto TrailingFence = TLI->emitTrailingFence(Builder, Order, IsStore, IsLoad); 18537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // The trailing fence is emitted before the instruction instead of after 18637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // because there is no easy way of setting Builder insertion point after 18737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // an instruction. So we must erase it from the BB, and insert it back 18837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // in the right place. 18937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // We have a guard here because not every atomic operation generates a 19037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // trailing fence. 19137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (TrailingFence) { 19237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TrailingFence->removeFromParent(); 19337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TrailingFence->insertAfter(I); 19437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 19537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 19637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return (LeadingFence || TrailingFence); 19737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 19837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 199f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// Get the iX type with the same bitwidth as T. 200f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga NainarIntegerType *AtomicExpand::getCorrespondingIntegerType(Type *T, 201f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const DataLayout &DL) { 202f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT VT = TLI->getValueType(DL, T); 203f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned BitWidth = VT.getStoreSizeInBits(); 204f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar assert(BitWidth == VT.getSizeInBits() && "must be a power of two"); 205f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return IntegerType::get(T->getContext(), BitWidth); 206f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 207f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 208f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// Convert an atomic load of a non-integral type to an integer load of the 209f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// equivelent bitwidth. See the function comment on 210f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// convertAtomicStoreToIntegerType for background. 211f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga NainarLoadInst *AtomicExpand::convertAtomicLoadToIntegerType(LoadInst *LI) { 212f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar auto *M = LI->getModule(); 213f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Type *NewTy = getCorrespondingIntegerType(LI->getType(), 214f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar M->getDataLayout()); 215f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 216f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar IRBuilder<> Builder(LI); 217f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 218f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *Addr = LI->getPointerOperand(); 219f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Type *PT = PointerType::get(NewTy, 220f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Addr->getType()->getPointerAddressSpace()); 221f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *NewAddr = Builder.CreateBitCast(Addr, PT); 222f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 223f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar auto *NewLI = Builder.CreateLoad(NewAddr); 224f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar NewLI->setAlignment(LI->getAlignment()); 225f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar NewLI->setVolatile(LI->isVolatile()); 226f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar NewLI->setAtomic(LI->getOrdering(), LI->getSynchScope()); 227f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DEBUG(dbgs() << "Replaced " << *LI << " with " << *NewLI << "\n"); 228f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 229f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *NewVal = Builder.CreateBitCast(NewLI, LI->getType()); 230f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar LI->replaceAllUsesWith(NewVal); 231f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar LI->eraseFromParent(); 232f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return NewLI; 233f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 234f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 235f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarbool AtomicExpand::tryExpandAtomicLoad(LoadInst *LI) { 236f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar switch (TLI->shouldExpandAtomicLoadInIR(LI)) { 237f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case TargetLoweringBase::AtomicExpansionKind::None: 238f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return false; 239f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case TargetLoweringBase::AtomicExpansionKind::LLSC: 240f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return expandAtomicOpToLLSC( 241f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar LI, LI->getPointerOperand(), LI->getOrdering(), 242f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar [](IRBuilder<> &Builder, Value *Loaded) { return Loaded; }); 243f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case TargetLoweringBase::AtomicExpansionKind::LLOnly: 24437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return expandAtomicLoadToLL(LI); 245f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case TargetLoweringBase::AtomicExpansionKind::CmpXChg: 24637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return expandAtomicLoadToCmpXchg(LI); 247f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 248f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar llvm_unreachable("Unhandled case in tryExpandAtomicLoad"); 24937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 25037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 25137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesbool AtomicExpand::expandAtomicLoadToLL(LoadInst *LI) { 25237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IRBuilder<> Builder(LI); 25337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 25437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // On some architectures, load-linked instructions are atomic for larger 25537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // sizes than normal loads. For example, the only 64-bit load guaranteed 25637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // to be single-copy atomic by ARM is an ldrexd (A3.5.3). 25737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Val = 25837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TLI->emitLoadLinked(Builder, LI->getPointerOperand(), LI->getOrdering()); 259f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar TLI->emitAtomicCmpXchgNoStoreLLBalance(Builder); 26037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 26137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines LI->replaceAllUsesWith(Val); 26237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines LI->eraseFromParent(); 26337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 26437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 26537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 26637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 26737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesbool AtomicExpand::expandAtomicLoadToCmpXchg(LoadInst *LI) { 26837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IRBuilder<> Builder(LI); 26937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AtomicOrdering Order = LI->getOrdering(); 27037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Addr = LI->getPointerOperand(); 27137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Type *Ty = cast<PointerType>(Addr->getType())->getElementType(); 27237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Constant *DummyVal = Constant::getNullValue(Ty); 27337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 27437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Pair = Builder.CreateAtomicCmpXchg( 27537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Addr, DummyVal, DummyVal, Order, 27637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AtomicCmpXchgInst::getStrongestFailureOrdering(Order)); 27737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Loaded = Builder.CreateExtractValue(Pair, 0, "loaded"); 27837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 27937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines LI->replaceAllUsesWith(Loaded); 28037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines LI->eraseFromParent(); 28137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 28237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 28337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 28437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 285f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// Convert an atomic store of a non-integral type to an integer store of the 286f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// equivelent bitwidth. We used to not support floating point or vector 287f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// atomics in the IR at all. The backends learned to deal with the bitcast 288f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// idiom because that was the only way of expressing the notion of a atomic 289f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// float or vector store. The long term plan is to teach each backend to 290f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// instruction select from the original atomic store, but as a migration 291f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// mechanism, we convert back to the old format which the backends understand. 292f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// Each backend will need individual work to recognize the new format. 293f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga NainarStoreInst *AtomicExpand::convertAtomicStoreToIntegerType(StoreInst *SI) { 294f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar IRBuilder<> Builder(SI); 295f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar auto *M = SI->getModule(); 296f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Type *NewTy = getCorrespondingIntegerType(SI->getValueOperand()->getType(), 297f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar M->getDataLayout()); 298f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *NewVal = Builder.CreateBitCast(SI->getValueOperand(), NewTy); 299f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 300f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *Addr = SI->getPointerOperand(); 301f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Type *PT = PointerType::get(NewTy, 302f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Addr->getType()->getPointerAddressSpace()); 303f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *NewAddr = Builder.CreateBitCast(Addr, PT); 304f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 305f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar StoreInst *NewSI = Builder.CreateStore(NewVal, NewAddr); 306f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar NewSI->setAlignment(SI->getAlignment()); 307f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar NewSI->setVolatile(SI->isVolatile()); 308f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar NewSI->setAtomic(SI->getOrdering(), SI->getSynchScope()); 309f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DEBUG(dbgs() << "Replaced " << *SI << " with " << *NewSI << "\n"); 310f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SI->eraseFromParent(); 311f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return NewSI; 312f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 313f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 31437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesbool AtomicExpand::expandAtomicStore(StoreInst *SI) { 31537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // This function is only called on atomic stores that are too large to be 31637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // atomic if implemented as a native store. So we replace them by an 31737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // atomic swap, that can be implemented for example as a ldrex/strex on ARM 31837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // or lock cmpxchg8/16b on X86, as these are atomic for larger sizes. 3194c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // It is the responsibility of the target to only signal expansion via 32037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // shouldExpandAtomicRMW in cases where this is required and possible. 32137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IRBuilder<> Builder(SI); 32237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AtomicRMWInst *AI = 32337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.CreateAtomicRMW(AtomicRMWInst::Xchg, SI->getPointerOperand(), 32437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SI->getValueOperand(), SI->getOrdering()); 32537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SI->eraseFromParent(); 32637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 32737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Now we have an appropriate swap instruction, lower it as usual. 3284c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar return tryExpandAtomicRMW(AI); 32937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 33037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 331f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarstatic void createCmpXchgInstFun(IRBuilder<> &Builder, Value *Addr, 332f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *Loaded, Value *NewVal, 333f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar AtomicOrdering MemOpOrder, 334f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *&Success, Value *&NewLoaded) { 335f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value* Pair = Builder.CreateAtomicCmpXchg( 336f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Addr, Loaded, NewVal, MemOpOrder, 337f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar AtomicCmpXchgInst::getStrongestFailureOrdering(MemOpOrder)); 338f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Success = Builder.CreateExtractValue(Pair, 1, "success"); 339f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded"); 34037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 34137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 34237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines/// Emit IR to implement the given atomicrmw operation on values in registers, 34337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines/// returning the new value. 34437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesstatic Value *performAtomicOp(AtomicRMWInst::BinOp Op, IRBuilder<> &Builder, 34537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Loaded, Value *Inc) { 34637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *NewVal; 34737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines switch (Op) { 34837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::Xchg: 34937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Inc; 35037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::Add: 35137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Builder.CreateAdd(Loaded, Inc, "new"); 35237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::Sub: 35337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Builder.CreateSub(Loaded, Inc, "new"); 35437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::And: 35537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Builder.CreateAnd(Loaded, Inc, "new"); 35637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::Nand: 35737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Builder.CreateNot(Builder.CreateAnd(Loaded, Inc), "new"); 35837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::Or: 35937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Builder.CreateOr(Loaded, Inc, "new"); 36037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::Xor: 36137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Builder.CreateXor(Loaded, Inc, "new"); 36237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::Max: 36337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines NewVal = Builder.CreateICmpSGT(Loaded, Inc); 36437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Builder.CreateSelect(NewVal, Loaded, Inc, "new"); 36537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::Min: 36637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines NewVal = Builder.CreateICmpSLE(Loaded, Inc); 36737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Builder.CreateSelect(NewVal, Loaded, Inc, "new"); 36837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::UMax: 36937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines NewVal = Builder.CreateICmpUGT(Loaded, Inc); 37037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Builder.CreateSelect(NewVal, Loaded, Inc, "new"); 37137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::UMin: 37237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines NewVal = Builder.CreateICmpULE(Loaded, Inc); 37337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Builder.CreateSelect(NewVal, Loaded, Inc, "new"); 37437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines default: 37537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines llvm_unreachable("Unknown atomic op"); 37637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 37737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 37837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 379f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarbool AtomicExpand::tryExpandAtomicRMW(AtomicRMWInst *AI) { 380f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar switch (TLI->shouldExpandAtomicRMWInIR(AI)) { 381f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case TargetLoweringBase::AtomicExpansionKind::None: 382f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return false; 383f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case TargetLoweringBase::AtomicExpansionKind::LLSC: 384f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return expandAtomicOpToLLSC(AI, AI->getPointerOperand(), AI->getOrdering(), 385f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar [&](IRBuilder<> &Builder, Value *Loaded) { 386f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return performAtomicOp(AI->getOperation(), 387f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Builder, Loaded, 388f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar AI->getValOperand()); 389f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar }); 390f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case TargetLoweringBase::AtomicExpansionKind::CmpXChg: 391f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return expandAtomicRMWToCmpXchg(AI, createCmpXchgInstFun); 392f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar default: 393f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar llvm_unreachable("Unhandled case in tryExpandAtomicRMW"); 394f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 395f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 396f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 397f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarbool AtomicExpand::expandAtomicOpToLLSC( 398f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Instruction *I, Value *Addr, AtomicOrdering MemOpOrder, 399f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar std::function<Value *(IRBuilder<> &, Value *)> PerformOp) { 400f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar BasicBlock *BB = I->getParent(); 40137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Function *F = BB->getParent(); 40237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines LLVMContext &Ctx = F->getContext(); 40337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 40437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Given: atomicrmw some_op iN* %addr, iN %incr ordering 40537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // 40637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // The standard expansion we produce is: 40737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // [...] 40837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // fence? 40937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // atomicrmw.start: 41037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // %loaded = @load.linked(%addr) 41137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // %new = some_op iN %loaded, %incr 41237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // %stored = @store_conditional(%new, %addr) 41337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // %try_again = icmp i32 ne %stored, 0 41437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // br i1 %try_again, label %loop, label %atomicrmw.end 41537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // atomicrmw.end: 41637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // fence? 41737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // [...] 418f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar BasicBlock *ExitBB = BB->splitBasicBlock(I->getIterator(), "atomicrmw.end"); 41937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines BasicBlock *LoopBB = BasicBlock::Create(Ctx, "atomicrmw.start", F, ExitBB); 42037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 421f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // This grabs the DebugLoc from I. 422f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar IRBuilder<> Builder(I); 42337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 42437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // The split call above "helpfully" added a branch at the end of BB (to the 42537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // wrong place), but we might want a fence too. It's easiest to just remove 42637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // the branch entirely. 42737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines std::prev(BB->end())->eraseFromParent(); 42837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.SetInsertPoint(BB); 42937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.CreateBr(LoopBB); 43037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 43137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Start the main loop block now that we've taken care of the preliminaries. 43237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.SetInsertPoint(LoopBB); 43337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Loaded = TLI->emitLoadLinked(Builder, Addr, MemOpOrder); 43437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 435f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *NewVal = PerformOp(Builder, Loaded); 43637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 43737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *StoreSuccess = 43837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TLI->emitStoreConditional(Builder, NewVal, Addr, MemOpOrder); 43937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *TryAgain = Builder.CreateICmpNE( 44037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines StoreSuccess, ConstantInt::get(IntegerType::get(Ctx, 32), 0), "tryagain"); 44137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.CreateCondBr(TryAgain, LoopBB, ExitBB); 44237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 44337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.SetInsertPoint(ExitBB, ExitBB->begin()); 44437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 445f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar I->replaceAllUsesWith(Loaded); 446f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar I->eraseFromParent(); 44737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 44837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 44937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 45037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 45137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesbool AtomicExpand::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) { 45237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AtomicOrdering SuccessOrder = CI->getSuccessOrdering(); 45337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AtomicOrdering FailureOrder = CI->getFailureOrdering(); 45437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Addr = CI->getPointerOperand(); 45537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines BasicBlock *BB = CI->getParent(); 45637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Function *F = BB->getParent(); 45737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines LLVMContext &Ctx = F->getContext(); 45837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // If getInsertFencesForAtomic() returns true, then the target does not want 45937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // to deal with memory orders, and emitLeading/TrailingFence should take care 46037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // of everything. Otherwise, emitLeading/TrailingFence are no-op and we 46137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // should preserve the ordering. 46237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AtomicOrdering MemOpOrder = 46337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TLI->getInsertFencesForAtomic() ? Monotonic : SuccessOrder; 46437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 46537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Given: cmpxchg some_op iN* %addr, iN %desired, iN %new success_ord fail_ord 46637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // 46737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // The full expansion we produce is: 46837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // [...] 46937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // fence? 47037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // cmpxchg.start: 47137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // %loaded = @load.linked(%addr) 47237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // %should_store = icmp eq %loaded, %desired 47337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // br i1 %should_store, label %cmpxchg.trystore, 474f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // label %cmpxchg.nostore 47537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // cmpxchg.trystore: 47637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // %stored = @store_conditional(%new, %addr) 47737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // %success = icmp eq i32 %stored, 0 47837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // br i1 %success, label %cmpxchg.success, label %loop/%cmpxchg.failure 47937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // cmpxchg.success: 48037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // fence? 48137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // br label %cmpxchg.end 482f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // cmpxchg.nostore: 483f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // @load_linked_fail_balance()? 484f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // br label %cmpxchg.failure 48537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // cmpxchg.failure: 48637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // fence? 48737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // br label %cmpxchg.end 48837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // cmpxchg.end: 48937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // %success = phi i1 [true, %cmpxchg.success], [false, %cmpxchg.failure] 49037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // %restmp = insertvalue { iN, i1 } undef, iN %loaded, 0 49137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // %res = insertvalue { iN, i1 } %restmp, i1 %success, 1 49237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // [...] 493f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar BasicBlock *ExitBB = BB->splitBasicBlock(CI->getIterator(), "cmpxchg.end"); 49437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines auto FailureBB = BasicBlock::Create(Ctx, "cmpxchg.failure", F, ExitBB); 495f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar auto NoStoreBB = BasicBlock::Create(Ctx, "cmpxchg.nostore", F, FailureBB); 496f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar auto SuccessBB = BasicBlock::Create(Ctx, "cmpxchg.success", F, NoStoreBB); 49737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines auto TryStoreBB = BasicBlock::Create(Ctx, "cmpxchg.trystore", F, SuccessBB); 49837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines auto LoopBB = BasicBlock::Create(Ctx, "cmpxchg.start", F, TryStoreBB); 49937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 50037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // This grabs the DebugLoc from CI 50137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IRBuilder<> Builder(CI); 50237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 50337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // The split call above "helpfully" added a branch at the end of BB (to the 50437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // wrong place), but we might want a fence too. It's easiest to just remove 50537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // the branch entirely. 50637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines std::prev(BB->end())->eraseFromParent(); 50737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.SetInsertPoint(BB); 50837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TLI->emitLeadingFence(Builder, SuccessOrder, /*IsStore=*/true, 50937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /*IsLoad=*/true); 51037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.CreateBr(LoopBB); 51137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 51237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Start the main loop block now that we've taken care of the preliminaries. 51337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.SetInsertPoint(LoopBB); 51437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Loaded = TLI->emitLoadLinked(Builder, Addr, MemOpOrder); 51537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *ShouldStore = 51637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.CreateICmpEQ(Loaded, CI->getCompareOperand(), "should_store"); 51737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 518f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // If the cmpxchg doesn't actually need any ordering when it fails, we can 51937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // jump straight past that fence instruction (if it exists). 520f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Builder.CreateCondBr(ShouldStore, TryStoreBB, NoStoreBB); 52137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 52237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.SetInsertPoint(TryStoreBB); 52337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *StoreSuccess = TLI->emitStoreConditional( 52437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder, CI->getNewValOperand(), Addr, MemOpOrder); 52537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines StoreSuccess = Builder.CreateICmpEQ( 52637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines StoreSuccess, ConstantInt::get(Type::getInt32Ty(Ctx), 0), "success"); 52737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.CreateCondBr(StoreSuccess, SuccessBB, 52837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CI->isWeak() ? FailureBB : LoopBB); 52937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 53037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Make sure later instructions don't get reordered with a fence if necessary. 53137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.SetInsertPoint(SuccessBB); 53237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TLI->emitTrailingFence(Builder, SuccessOrder, /*IsStore=*/true, 53337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /*IsLoad=*/true); 53437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.CreateBr(ExitBB); 53537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 536f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Builder.SetInsertPoint(NoStoreBB); 537f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // In the failing case, where we don't execute the store-conditional, the 538f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // target might want to balance out the load-linked with a dedicated 539f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // instruction (e.g., on ARM, clearing the exclusive monitor). 540f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar TLI->emitAtomicCmpXchgNoStoreLLBalance(Builder); 541f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Builder.CreateBr(FailureBB); 542f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 54337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.SetInsertPoint(FailureBB); 54437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TLI->emitTrailingFence(Builder, FailureOrder, /*IsStore=*/true, 54537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /*IsLoad=*/true); 54637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.CreateBr(ExitBB); 54737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 54837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Finally, we have control-flow based knowledge of whether the cmpxchg 54937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // succeeded or not. We expose this to later passes by converting any 55037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // subsequent "icmp eq/ne %loaded, %oldval" into a use of an appropriate PHI. 55137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 55237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Setup the builder so we can create any PHIs we need. 55337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.SetInsertPoint(ExitBB, ExitBB->begin()); 55437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines PHINode *Success = Builder.CreatePHI(Type::getInt1Ty(Ctx), 2); 55537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Success->addIncoming(ConstantInt::getTrue(Ctx), SuccessBB); 55637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Success->addIncoming(ConstantInt::getFalse(Ctx), FailureBB); 55737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 55837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Look for any users of the cmpxchg that are just comparing the loaded value 55937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // against the desired one, and replace them with the CFG-derived version. 56037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SmallVector<ExtractValueInst *, 2> PrunedInsts; 56137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines for (auto User : CI->users()) { 56237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ExtractValueInst *EV = dyn_cast<ExtractValueInst>(User); 56337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!EV) 56437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines continue; 56537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 56637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines assert(EV->getNumIndices() == 1 && EV->getIndices()[0] <= 1 && 56737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines "weird extraction from { iN, i1 }"); 56837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 56937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (EV->getIndices()[0] == 0) 57037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EV->replaceAllUsesWith(Loaded); 57137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines else 57237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EV->replaceAllUsesWith(Success); 57337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 57437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines PrunedInsts.push_back(EV); 57537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 57637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 57737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // We can remove the instructions now we're no longer iterating through them. 57837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines for (auto EV : PrunedInsts) 57937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EV->eraseFromParent(); 58037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 58137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!CI->use_empty()) { 58237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Some use of the full struct return that we don't understand has happened, 58337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // so we've got to reconstruct it properly. 58437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Res; 58537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Res = Builder.CreateInsertValue(UndefValue::get(CI->getType()), Loaded, 0); 58637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Res = Builder.CreateInsertValue(Res, Success, 1); 58737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 58837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CI->replaceAllUsesWith(Res); 58937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 59037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 59137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CI->eraseFromParent(); 59237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 59337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 59437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 59537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesbool AtomicExpand::isIdempotentRMW(AtomicRMWInst* RMWI) { 59637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines auto C = dyn_cast<ConstantInt>(RMWI->getValOperand()); 59737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if(!C) 59837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 59937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 60037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AtomicRMWInst::BinOp Op = RMWI->getOperation(); 60137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines switch(Op) { 60237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::Add: 60337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::Sub: 60437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::Or: 60537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::Xor: 60637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return C->isZero(); 60737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::And: 60837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return C->isMinusOne(); 60937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // FIXME: we could also treat Min/Max/UMin/UMax by the INT_MIN/INT_MAX/... 61037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines default: 61137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 61237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 61337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 61437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 61537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesbool AtomicExpand::simplifyIdempotentRMW(AtomicRMWInst* RMWI) { 61637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (auto ResultingLoad = TLI->lowerIdempotentRMWIntoFencedLoad(RMWI)) { 617f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar tryExpandAtomicLoad(ResultingLoad); 61837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 61937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 62037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 62137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 622f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 623f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarbool llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI, 624f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar CreateCmpXchgInstFun CreateCmpXchg) { 625f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar assert(AI); 626f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 627f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar AtomicOrdering MemOpOrder = 628f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar AI->getOrdering() == Unordered ? Monotonic : AI->getOrdering(); 629f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *Addr = AI->getPointerOperand(); 630f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar BasicBlock *BB = AI->getParent(); 631f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Function *F = BB->getParent(); 632f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar LLVMContext &Ctx = F->getContext(); 633f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 634f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Given: atomicrmw some_op iN* %addr, iN %incr ordering 635f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // 636f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // The standard expansion we produce is: 637f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // [...] 638f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // %init_loaded = load atomic iN* %addr 639f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // br label %loop 640f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // loop: 641f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // %loaded = phi iN [ %init_loaded, %entry ], [ %new_loaded, %loop ] 642f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // %new = some_op iN %loaded, %incr 643f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // %pair = cmpxchg iN* %addr, iN %loaded, iN %new 644f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // %new_loaded = extractvalue { iN, i1 } %pair, 0 645f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // %success = extractvalue { iN, i1 } %pair, 1 646f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // br i1 %success, label %atomicrmw.end, label %loop 647f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // atomicrmw.end: 648f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // [...] 649f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar BasicBlock *ExitBB = BB->splitBasicBlock(AI->getIterator(), "atomicrmw.end"); 650f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar BasicBlock *LoopBB = BasicBlock::Create(Ctx, "atomicrmw.start", F, ExitBB); 651f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 652f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // This grabs the DebugLoc from AI. 653f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar IRBuilder<> Builder(AI); 654f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 655f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // The split call above "helpfully" added a branch at the end of BB (to the 656f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // wrong place), but we want a load. It's easiest to just remove 657f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // the branch entirely. 658f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar std::prev(BB->end())->eraseFromParent(); 659f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Builder.SetInsertPoint(BB); 660f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar LoadInst *InitLoaded = Builder.CreateLoad(Addr); 661f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Atomics require at least natural alignment. 662f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar InitLoaded->setAlignment(AI->getType()->getPrimitiveSizeInBits() / 8); 663f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Builder.CreateBr(LoopBB); 664f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 665f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Start the main loop block now that we've taken care of the preliminaries. 666f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Builder.SetInsertPoint(LoopBB); 667f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar PHINode *Loaded = Builder.CreatePHI(AI->getType(), 2, "loaded"); 668f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Loaded->addIncoming(InitLoaded, BB); 669f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 670f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *NewVal = 671f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar performAtomicOp(AI->getOperation(), Builder, Loaded, AI->getValOperand()); 672f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 673f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *NewLoaded = nullptr; 674f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *Success = nullptr; 675f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 676f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar CreateCmpXchg(Builder, Addr, Loaded, NewVal, MemOpOrder, 677f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Success, NewLoaded); 678f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar assert(Success && NewLoaded); 679f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 680f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Loaded->addIncoming(NewLoaded, LoopBB); 681f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 682f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Builder.CreateCondBr(Success, ExitBB, LoopBB); 683f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 684f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Builder.SetInsertPoint(ExitBB, ExitBB->begin()); 685f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 686f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar AI->replaceAllUsesWith(NewLoaded); 687f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar AI->eraseFromParent(); 688f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 689f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return true; 690f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 691