137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines//===-- AtomicExpandPass.cpp - Expand atomic instructions -------===// 237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// 337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// The LLVM Compiler Infrastructure 437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// 537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// This file is distributed under the University of Illinois Open Source 637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// License. See LICENSE.TXT for details. 737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// 837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines//===----------------------------------------------------------------------===// 937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// 1037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// This file contains a pass (at IR level) to replace atomic instructions with 11de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// __atomic_* library calls, or target specific instruction which implement the 12de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// same semantics in a way which better fits the target backend. This can 13de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// include the use of (intrinsic-based) load-linked/store-conditional loops, 14de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// AtomicCmpXchg, or type coercions. 1537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines// 1637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines//===----------------------------------------------------------------------===// 1737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 18f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar#include "llvm/CodeGen/AtomicExpandUtils.h" 1937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "llvm/CodeGen/Passes.h" 2037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "llvm/IR/Function.h" 2137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "llvm/IR/IRBuilder.h" 2237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "llvm/IR/InstIterator.h" 2337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "llvm/IR/Instructions.h" 2437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "llvm/IR/Intrinsics.h" 2537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "llvm/IR/Module.h" 2637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "llvm/Support/Debug.h" 27f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar#include "llvm/Support/raw_ostream.h" 2837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "llvm/Target/TargetLowering.h" 2937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "llvm/Target/TargetMachine.h" 3037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "llvm/Target/TargetSubtargetInfo.h" 3137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 3237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesusing namespace llvm; 3337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 3437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#define DEBUG_TYPE "atomic-expand" 3537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 3637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesnamespace { 3737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines class AtomicExpand: public FunctionPass { 3837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines const TargetMachine *TM; 39ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const TargetLowering *TLI; 4037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines public: 4137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines static char ID; // Pass identification, replacement for typeid 4237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines explicit AtomicExpand(const TargetMachine *TM = nullptr) 43ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines : FunctionPass(ID), TM(TM), TLI(nullptr) { 4437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines initializeAtomicExpandPass(*PassRegistry::getPassRegistry()); 4537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 4637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 4737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool runOnFunction(Function &F) override; 4837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 4937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines private: 5037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool bracketInstWithFences(Instruction *I, AtomicOrdering Order, 5137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool IsStore, bool IsLoad); 52f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar IntegerType *getCorrespondingIntegerType(Type *T, const DataLayout &DL); 53f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar LoadInst *convertAtomicLoadToIntegerType(LoadInst *LI); 54f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar bool tryExpandAtomicLoad(LoadInst *LI); 5537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool expandAtomicLoadToLL(LoadInst *LI); 5637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool expandAtomicLoadToCmpXchg(LoadInst *LI); 57f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar StoreInst *convertAtomicStoreToIntegerType(StoreInst *SI); 5837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool expandAtomicStore(StoreInst *SI); 594c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar bool tryExpandAtomicRMW(AtomicRMWInst *AI); 60de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value * 61de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar insertRMWLLSCLoop(IRBuilder<> &Builder, Type *ResultTy, Value *Addr, 62de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AtomicOrdering MemOpOrder, 63de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar function_ref<Value *(IRBuilder<> &, Value *)> PerformOp); 64de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar void expandAtomicOpToLLSC( 65de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Instruction *I, Type *ResultTy, Value *Addr, AtomicOrdering MemOpOrder, 66de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar function_ref<Value *(IRBuilder<> &, Value *)> PerformOp); 67de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar void expandPartwordAtomicRMW( 68de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AtomicRMWInst *I, 69de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar TargetLoweringBase::AtomicExpansionKind ExpansionKind); 70de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar void expandPartwordCmpXchg(AtomicCmpXchgInst *I); 71de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 72de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AtomicCmpXchgInst *convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI); 73de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar static Value *insertRMWCmpXchgLoop( 74de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar IRBuilder<> &Builder, Type *ResultType, Value *Addr, 75de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AtomicOrdering MemOpOrder, 76de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar function_ref<Value *(IRBuilder<> &, Value *)> PerformOp, 77de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CreateCmpXchgInstFun CreateCmpXchg); 78de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 7937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool expandAtomicCmpXchg(AtomicCmpXchgInst *CI); 8037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool isIdempotentRMW(AtomicRMWInst *AI); 8137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool simplifyIdempotentRMW(AtomicRMWInst *AI); 82de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 83de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool expandAtomicOpToLibcall(Instruction *I, unsigned Size, unsigned Align, 84de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *PointerOperand, Value *ValueOperand, 85de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *CASExpected, AtomicOrdering Ordering, 86de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AtomicOrdering Ordering2, 87de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ArrayRef<RTLIB::Libcall> Libcalls); 88de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar void expandAtomicLoadToLibcall(LoadInst *LI); 89de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar void expandAtomicStoreToLibcall(StoreInst *LI); 90de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar void expandAtomicRMWToLibcall(AtomicRMWInst *I); 91de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar void expandAtomicCASToLibcall(AtomicCmpXchgInst *I); 92de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 93de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar friend bool 94de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI, 95de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CreateCmpXchgInstFun CreateCmpXchg); 9637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines }; 9737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 9837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 9937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hineschar AtomicExpand::ID = 0; 10037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hineschar &llvm::AtomicExpandID = AtomicExpand::ID; 101de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarINITIALIZE_TM_PASS(AtomicExpand, "atomic-expand", "Expand Atomic instructions", 102de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar false, false) 10337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 10437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen HinesFunctionPass *llvm::createAtomicExpandPass(const TargetMachine *TM) { 10537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return new AtomicExpand(TM); 10637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 10737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 108de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarnamespace { 109de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// Helper functions to retrieve the size of atomic instructions. 110de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarunsigned getAtomicOpSize(LoadInst *LI) { 111de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const DataLayout &DL = LI->getModule()->getDataLayout(); 112de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return DL.getTypeStoreSize(LI->getType()); 113de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 114de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 115de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarunsigned getAtomicOpSize(StoreInst *SI) { 116de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const DataLayout &DL = SI->getModule()->getDataLayout(); 117de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return DL.getTypeStoreSize(SI->getValueOperand()->getType()); 118de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 119de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 120de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarunsigned getAtomicOpSize(AtomicRMWInst *RMWI) { 121de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const DataLayout &DL = RMWI->getModule()->getDataLayout(); 122de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return DL.getTypeStoreSize(RMWI->getValOperand()->getType()); 123de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 124de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 125de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarunsigned getAtomicOpSize(AtomicCmpXchgInst *CASI) { 126de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const DataLayout &DL = CASI->getModule()->getDataLayout(); 127de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return DL.getTypeStoreSize(CASI->getCompareOperand()->getType()); 128de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 129de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 130de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// Helper functions to retrieve the alignment of atomic instructions. 131de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarunsigned getAtomicOpAlign(LoadInst *LI) { 132de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned Align = LI->getAlignment(); 133de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // In the future, if this IR restriction is relaxed, we should 134de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // return DataLayout::getABITypeAlignment when there's no align 135de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // value. 136de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert(Align != 0 && "An atomic LoadInst always has an explicit alignment"); 137de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return Align; 138de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 139de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 140de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarunsigned getAtomicOpAlign(StoreInst *SI) { 141de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned Align = SI->getAlignment(); 142de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // In the future, if this IR restriction is relaxed, we should 143de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // return DataLayout::getABITypeAlignment when there's no align 144de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // value. 145de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert(Align != 0 && "An atomic StoreInst always has an explicit alignment"); 146de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return Align; 147de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 148de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 149de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarunsigned getAtomicOpAlign(AtomicRMWInst *RMWI) { 150de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // TODO(PR27168): This instruction has no alignment attribute, but unlike the 151de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // default alignment for load/store, the default here is to assume 152de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // it has NATURAL alignment, not DataLayout-specified alignment. 153de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const DataLayout &DL = RMWI->getModule()->getDataLayout(); 154de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return DL.getTypeStoreSize(RMWI->getValOperand()->getType()); 155de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 156de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 157de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarunsigned getAtomicOpAlign(AtomicCmpXchgInst *CASI) { 158de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // TODO(PR27168): same comment as above. 159de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const DataLayout &DL = CASI->getModule()->getDataLayout(); 160de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return DL.getTypeStoreSize(CASI->getCompareOperand()->getType()); 161de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 162de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 163de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// Determine if a particular atomic operation has a supported size, 164de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// and is of appropriate alignment, to be passed through for target 165de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// lowering. (Versus turning into a __atomic libcall) 166de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainartemplate <typename Inst> 167de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarbool atomicSizeSupported(const TargetLowering *TLI, Inst *I) { 168de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned Size = getAtomicOpSize(I); 169de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned Align = getAtomicOpAlign(I); 170de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return Align >= Size && Size <= TLI->getMaxAtomicSizeInBitsSupported() / 8; 171de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 172de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 173de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} // end anonymous namespace 174de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 17537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesbool AtomicExpand::runOnFunction(Function &F) { 176ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!TM || !TM->getSubtargetImpl(F)->enableAtomicExpand()) 17737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 178ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines TLI = TM->getSubtargetImpl(F)->getTargetLowering(); 17937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 18037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SmallVector<Instruction *, 1> AtomicInsts; 18137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 18237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Changing control-flow while iterating through it is a bad idea, so gather a 18337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // list of all atomic instructions before we start. 184de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar for (inst_iterator II = inst_begin(F), E = inst_end(F); II != E; ++II) { 185de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Instruction *I = &*II; 186de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (I->isAtomic() && !isa<FenceInst>(I)) 187de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AtomicInsts.push_back(I); 18837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 18937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 19037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool MadeChange = false; 19137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines for (auto I : AtomicInsts) { 19237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines auto LI = dyn_cast<LoadInst>(I); 19337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines auto SI = dyn_cast<StoreInst>(I); 19437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines auto RMWI = dyn_cast<AtomicRMWInst>(I); 19537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines auto CASI = dyn_cast<AtomicCmpXchgInst>(I); 196de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert((LI || SI || RMWI || CASI) && "Unknown atomic instruction"); 197de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 198de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // If the Size/Alignment is not supported, replace with a libcall. 199de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (LI) { 200de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!atomicSizeSupported(TLI, LI)) { 201de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar expandAtomicLoadToLibcall(LI); 202de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MadeChange = true; 203de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar continue; 204de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 205de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else if (SI) { 206de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!atomicSizeSupported(TLI, SI)) { 207de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar expandAtomicStoreToLibcall(SI); 208de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MadeChange = true; 209de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar continue; 210de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 211de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else if (RMWI) { 212de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!atomicSizeSupported(TLI, RMWI)) { 213de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar expandAtomicRMWToLibcall(RMWI); 214de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MadeChange = true; 215de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar continue; 216de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 217de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else if (CASI) { 218de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!atomicSizeSupported(TLI, CASI)) { 219de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar expandAtomicCASToLibcall(CASI); 220de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MadeChange = true; 221de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar continue; 222de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 223de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 22437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 225de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (TLI->shouldInsertFencesForAtomic(I)) { 226de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar auto FenceOrdering = AtomicOrdering::Monotonic; 227de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool IsStore, IsLoad; 228de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (LI && isAcquireOrStronger(LI->getOrdering())) { 22937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines FenceOrdering = LI->getOrdering(); 230de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar LI->setOrdering(AtomicOrdering::Monotonic); 23137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IsStore = false; 23237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IsLoad = true; 233de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else if (SI && isReleaseOrStronger(SI->getOrdering())) { 23437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines FenceOrdering = SI->getOrdering(); 235de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SI->setOrdering(AtomicOrdering::Monotonic); 23637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IsStore = true; 23737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IsLoad = false; 238de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else if (RMWI && (isReleaseOrStronger(RMWI->getOrdering()) || 239de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar isAcquireOrStronger(RMWI->getOrdering()))) { 24037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines FenceOrdering = RMWI->getOrdering(); 241de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RMWI->setOrdering(AtomicOrdering::Monotonic); 24237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IsStore = IsLoad = true; 243f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } else if (CASI && !TLI->shouldExpandAtomicCmpXchgInIR(CASI) && 244de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar (isReleaseOrStronger(CASI->getSuccessOrdering()) || 245de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar isAcquireOrStronger(CASI->getSuccessOrdering()))) { 24637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // If a compare and swap is lowered to LL/SC, we can do smarter fence 24737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // insertion, with a stronger one on the success path than on the 24837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // failure path. As a result, fence insertion is directly done by 24937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // expandAtomicCmpXchg in that case. 25037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines FenceOrdering = CASI->getSuccessOrdering(); 251de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CASI->setSuccessOrdering(AtomicOrdering::Monotonic); 252de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CASI->setFailureOrdering(AtomicOrdering::Monotonic); 25337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IsStore = IsLoad = true; 25437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 25537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 256de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (FenceOrdering != AtomicOrdering::Monotonic) { 25737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines MadeChange |= bracketInstWithFences(I, FenceOrdering, IsStore, IsLoad); 25837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 25937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 26037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 261f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (LI) { 262f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (LI->getType()->isFloatingPointTy()) { 263f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // TODO: add a TLI hook to control this so that each target can 264f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // convert to lowering the original type one at a time. 265f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar LI = convertAtomicLoadToIntegerType(LI); 266f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar assert(LI->getType()->isIntegerTy() && "invariant broken"); 267f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MadeChange = true; 268f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 269de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 270f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MadeChange |= tryExpandAtomicLoad(LI); 271f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } else if (SI) { 272f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (SI->getValueOperand()->getType()->isFloatingPointTy()) { 273f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // TODO: add a TLI hook to control this so that each target can 274f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // convert to lowering the original type one at a time. 275f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SI = convertAtomicStoreToIntegerType(SI); 276f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar assert(SI->getValueOperand()->getType()->isIntegerTy() && 277f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar "invariant broken"); 278f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MadeChange = true; 279f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 280f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 281f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar if (TLI->shouldExpandAtomicStoreInIR(SI)) 282f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar MadeChange |= expandAtomicStore(SI); 28337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } else if (RMWI) { 28437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // There are two different ways of expanding RMW instructions: 28537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // - into a load if it is idempotent 28637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // - into a Cmpxchg/LL-SC loop otherwise 28737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // we try them in that order. 2884c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar 2894c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar if (isIdempotentRMW(RMWI) && simplifyIdempotentRMW(RMWI)) { 2904c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar MadeChange = true; 2914c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar } else { 2924c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar MadeChange |= tryExpandAtomicRMW(RMWI); 2934c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar } 294de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else if (CASI) { 295de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // TODO: when we're ready to make the change at the IR level, we can 296de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // extend convertCmpXchgToInteger for floating point too. 297de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert(!CASI->getCompareOperand()->getType()->isFloatingPointTy() && 298de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar "unimplemented - floating point not legal at IR level"); 299de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (CASI->getCompareOperand()->getType()->isPointerTy() ) { 300de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // TODO: add a TLI hook to control this so that each target can 301de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // convert to lowering the original type one at a time. 302de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CASI = convertCmpXchgToIntegerType(CASI); 303de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert(CASI->getCompareOperand()->getType()->isIntegerTy() && 304de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar "invariant broken"); 305de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MadeChange = true; 306de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 307de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 308de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8; 309de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned ValueSize = getAtomicOpSize(CASI); 310de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (ValueSize < MinCASSize) { 311de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert(!TLI->shouldExpandAtomicCmpXchgInIR(CASI) && 312de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar "MinCmpXchgSizeInBits not yet supported for LL/SC expansions."); 313de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar expandPartwordCmpXchg(CASI); 314de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else { 315de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (TLI->shouldExpandAtomicCmpXchgInIR(CASI)) 316de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MadeChange |= expandAtomicCmpXchg(CASI); 317de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 31837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 31937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 32037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return MadeChange; 32137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 32237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 32337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesbool AtomicExpand::bracketInstWithFences(Instruction *I, AtomicOrdering Order, 32437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool IsStore, bool IsLoad) { 32537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IRBuilder<> Builder(I); 32637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 327ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines auto LeadingFence = TLI->emitLeadingFence(Builder, Order, IsStore, IsLoad); 32837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 329ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines auto TrailingFence = TLI->emitTrailingFence(Builder, Order, IsStore, IsLoad); 33037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // The trailing fence is emitted before the instruction instead of after 33137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // because there is no easy way of setting Builder insertion point after 33237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // an instruction. So we must erase it from the BB, and insert it back 33337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // in the right place. 33437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // We have a guard here because not every atomic operation generates a 33537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // trailing fence. 33637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (TrailingFence) { 33737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TrailingFence->removeFromParent(); 33837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TrailingFence->insertAfter(I); 33937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 34037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 34137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return (LeadingFence || TrailingFence); 34237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 34337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 344f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// Get the iX type with the same bitwidth as T. 345f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga NainarIntegerType *AtomicExpand::getCorrespondingIntegerType(Type *T, 346f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const DataLayout &DL) { 347f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar EVT VT = TLI->getValueType(DL, T); 348f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar unsigned BitWidth = VT.getStoreSizeInBits(); 349f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar assert(BitWidth == VT.getSizeInBits() && "must be a power of two"); 350f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return IntegerType::get(T->getContext(), BitWidth); 351f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 352f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 353f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// Convert an atomic load of a non-integral type to an integer load of the 354de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// equivalent bitwidth. See the function comment on 355f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// convertAtomicStoreToIntegerType for background. 356f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga NainarLoadInst *AtomicExpand::convertAtomicLoadToIntegerType(LoadInst *LI) { 357f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar auto *M = LI->getModule(); 358f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Type *NewTy = getCorrespondingIntegerType(LI->getType(), 359f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar M->getDataLayout()); 360f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 361f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar IRBuilder<> Builder(LI); 362f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 363f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *Addr = LI->getPointerOperand(); 364f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Type *PT = PointerType::get(NewTy, 365f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Addr->getType()->getPointerAddressSpace()); 366f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *NewAddr = Builder.CreateBitCast(Addr, PT); 367f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 368f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar auto *NewLI = Builder.CreateLoad(NewAddr); 369f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar NewLI->setAlignment(LI->getAlignment()); 370f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar NewLI->setVolatile(LI->isVolatile()); 371f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar NewLI->setAtomic(LI->getOrdering(), LI->getSynchScope()); 372f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DEBUG(dbgs() << "Replaced " << *LI << " with " << *NewLI << "\n"); 373f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 374f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *NewVal = Builder.CreateBitCast(NewLI, LI->getType()); 375f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar LI->replaceAllUsesWith(NewVal); 376f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar LI->eraseFromParent(); 377f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return NewLI; 378f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 379f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 380f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarbool AtomicExpand::tryExpandAtomicLoad(LoadInst *LI) { 381f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar switch (TLI->shouldExpandAtomicLoadInIR(LI)) { 382f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case TargetLoweringBase::AtomicExpansionKind::None: 383f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return false; 384f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case TargetLoweringBase::AtomicExpansionKind::LLSC: 385de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar expandAtomicOpToLLSC( 386de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar LI, LI->getType(), LI->getPointerOperand(), LI->getOrdering(), 387f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar [](IRBuilder<> &Builder, Value *Loaded) { return Loaded; }); 388de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return true; 389f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case TargetLoweringBase::AtomicExpansionKind::LLOnly: 39037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return expandAtomicLoadToLL(LI); 391f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case TargetLoweringBase::AtomicExpansionKind::CmpXChg: 39237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return expandAtomicLoadToCmpXchg(LI); 393f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 394f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar llvm_unreachable("Unhandled case in tryExpandAtomicLoad"); 39537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 39637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 39737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesbool AtomicExpand::expandAtomicLoadToLL(LoadInst *LI) { 39837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IRBuilder<> Builder(LI); 39937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 40037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // On some architectures, load-linked instructions are atomic for larger 40137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // sizes than normal loads. For example, the only 64-bit load guaranteed 40237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // to be single-copy atomic by ARM is an ldrexd (A3.5.3). 40337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Val = 40437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TLI->emitLoadLinked(Builder, LI->getPointerOperand(), LI->getOrdering()); 405f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar TLI->emitAtomicCmpXchgNoStoreLLBalance(Builder); 40637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 40737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines LI->replaceAllUsesWith(Val); 40837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines LI->eraseFromParent(); 40937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 41037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 41137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 41237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 41337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesbool AtomicExpand::expandAtomicLoadToCmpXchg(LoadInst *LI) { 41437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IRBuilder<> Builder(LI); 41537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AtomicOrdering Order = LI->getOrdering(); 41637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Addr = LI->getPointerOperand(); 41737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Type *Ty = cast<PointerType>(Addr->getType())->getElementType(); 41837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Constant *DummyVal = Constant::getNullValue(Ty); 41937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 42037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Pair = Builder.CreateAtomicCmpXchg( 42137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Addr, DummyVal, DummyVal, Order, 42237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AtomicCmpXchgInst::getStrongestFailureOrdering(Order)); 42337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Loaded = Builder.CreateExtractValue(Pair, 0, "loaded"); 42437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 42537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines LI->replaceAllUsesWith(Loaded); 42637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines LI->eraseFromParent(); 42737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 42837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 42937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 43037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 431f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// Convert an atomic store of a non-integral type to an integer store of the 432de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// equivalent bitwidth. We used to not support floating point or vector 433f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// atomics in the IR at all. The backends learned to deal with the bitcast 434f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// idiom because that was the only way of expressing the notion of a atomic 435f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// float or vector store. The long term plan is to teach each backend to 436f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// instruction select from the original atomic store, but as a migration 437f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// mechanism, we convert back to the old format which the backends understand. 438f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar/// Each backend will need individual work to recognize the new format. 439f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga NainarStoreInst *AtomicExpand::convertAtomicStoreToIntegerType(StoreInst *SI) { 440f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar IRBuilder<> Builder(SI); 441f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar auto *M = SI->getModule(); 442f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Type *NewTy = getCorrespondingIntegerType(SI->getValueOperand()->getType(), 443f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar M->getDataLayout()); 444f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *NewVal = Builder.CreateBitCast(SI->getValueOperand(), NewTy); 445f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 446f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *Addr = SI->getPointerOperand(); 447f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Type *PT = PointerType::get(NewTy, 448f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Addr->getType()->getPointerAddressSpace()); 449f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *NewAddr = Builder.CreateBitCast(Addr, PT); 450f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 451f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar StoreInst *NewSI = Builder.CreateStore(NewVal, NewAddr); 452f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar NewSI->setAlignment(SI->getAlignment()); 453f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar NewSI->setVolatile(SI->isVolatile()); 454f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar NewSI->setAtomic(SI->getOrdering(), SI->getSynchScope()); 455f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar DEBUG(dbgs() << "Replaced " << *SI << " with " << *NewSI << "\n"); 456f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SI->eraseFromParent(); 457f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return NewSI; 458f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 459f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 46037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesbool AtomicExpand::expandAtomicStore(StoreInst *SI) { 46137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // This function is only called on atomic stores that are too large to be 46237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // atomic if implemented as a native store. So we replace them by an 46337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // atomic swap, that can be implemented for example as a ldrex/strex on ARM 46437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // or lock cmpxchg8/16b on X86, as these are atomic for larger sizes. 4654c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // It is the responsibility of the target to only signal expansion via 46637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // shouldExpandAtomicRMW in cases where this is required and possible. 46737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IRBuilder<> Builder(SI); 46837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AtomicRMWInst *AI = 46937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.CreateAtomicRMW(AtomicRMWInst::Xchg, SI->getPointerOperand(), 47037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SI->getValueOperand(), SI->getOrdering()); 47137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SI->eraseFromParent(); 47237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 47337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Now we have an appropriate swap instruction, lower it as usual. 4744c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar return tryExpandAtomicRMW(AI); 47537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 47637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 477f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarstatic void createCmpXchgInstFun(IRBuilder<> &Builder, Value *Addr, 478f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *Loaded, Value *NewVal, 479f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar AtomicOrdering MemOpOrder, 480f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *&Success, Value *&NewLoaded) { 481f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value* Pair = Builder.CreateAtomicCmpXchg( 482f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Addr, Loaded, NewVal, MemOpOrder, 483f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar AtomicCmpXchgInst::getStrongestFailureOrdering(MemOpOrder)); 484f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Success = Builder.CreateExtractValue(Pair, 1, "success"); 485f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded"); 48637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 48737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 48837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines/// Emit IR to implement the given atomicrmw operation on values in registers, 48937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines/// returning the new value. 49037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesstatic Value *performAtomicOp(AtomicRMWInst::BinOp Op, IRBuilder<> &Builder, 49137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Loaded, Value *Inc) { 49237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *NewVal; 49337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines switch (Op) { 49437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::Xchg: 49537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Inc; 49637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::Add: 49737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Builder.CreateAdd(Loaded, Inc, "new"); 49837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::Sub: 49937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Builder.CreateSub(Loaded, Inc, "new"); 50037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::And: 50137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Builder.CreateAnd(Loaded, Inc, "new"); 50237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::Nand: 50337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Builder.CreateNot(Builder.CreateAnd(Loaded, Inc), "new"); 50437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::Or: 50537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Builder.CreateOr(Loaded, Inc, "new"); 50637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::Xor: 50737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Builder.CreateXor(Loaded, Inc, "new"); 50837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::Max: 50937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines NewVal = Builder.CreateICmpSGT(Loaded, Inc); 51037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Builder.CreateSelect(NewVal, Loaded, Inc, "new"); 51137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::Min: 51237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines NewVal = Builder.CreateICmpSLE(Loaded, Inc); 51337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Builder.CreateSelect(NewVal, Loaded, Inc, "new"); 51437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::UMax: 51537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines NewVal = Builder.CreateICmpUGT(Loaded, Inc); 51637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Builder.CreateSelect(NewVal, Loaded, Inc, "new"); 51737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::UMin: 51837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines NewVal = Builder.CreateICmpULE(Loaded, Inc); 51937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Builder.CreateSelect(NewVal, Loaded, Inc, "new"); 52037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines default: 52137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines llvm_unreachable("Unknown atomic op"); 52237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 52337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 52437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 525f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainarbool AtomicExpand::tryExpandAtomicRMW(AtomicRMWInst *AI) { 526f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar switch (TLI->shouldExpandAtomicRMWInIR(AI)) { 527f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar case TargetLoweringBase::AtomicExpansionKind::None: 528f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return false; 529de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case TargetLoweringBase::AtomicExpansionKind::LLSC: { 530de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8; 531de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned ValueSize = getAtomicOpSize(AI); 532de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (ValueSize < MinCASSize) { 533de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar llvm_unreachable( 534de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar "MinCmpXchgSizeInBits not yet supported for LL/SC architectures."); 535de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else { 536de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar auto PerformOp = [&](IRBuilder<> &Builder, Value *Loaded) { 537de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return performAtomicOp(AI->getOperation(), Builder, Loaded, 538de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AI->getValOperand()); 539de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar }; 540de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar expandAtomicOpToLLSC(AI, AI->getType(), AI->getPointerOperand(), 541de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AI->getOrdering(), PerformOp); 542de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 543de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return true; 544de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 545de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case TargetLoweringBase::AtomicExpansionKind::CmpXChg: { 546de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8; 547de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned ValueSize = getAtomicOpSize(AI); 548de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (ValueSize < MinCASSize) { 549de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar expandPartwordAtomicRMW(AI, 550de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar TargetLoweringBase::AtomicExpansionKind::CmpXChg); 551de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else { 552de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar expandAtomicRMWToCmpXchg(AI, createCmpXchgInstFun); 553de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 554de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return true; 555de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 556f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar default: 557f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar llvm_unreachable("Unhandled case in tryExpandAtomicRMW"); 558f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar } 559f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 560f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 561de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarnamespace { 562de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 563de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// Result values from createMaskInstrs helper. 564de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstruct PartwordMaskValues { 565de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Type *WordType; 566de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Type *ValueType; 567de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *AlignedAddr; 568de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *ShiftAmt; 569de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *Mask; 570de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *Inv_Mask; 571de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar}; 572de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} // end anonymous namespace 573de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 574de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// This is a helper function which builds instructions to provide 575de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// values necessary for partword atomic operations. It takes an 576de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// incoming address, Addr, and ValueType, and constructs the address, 577de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// shift-amounts and masks needed to work with a larger value of size 578de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// WordSize. 579de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// 580de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// AlignedAddr: Addr rounded down to a multiple of WordSize 581de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// 582de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// ShiftAmt: Number of bits to right-shift a WordSize value loaded 583de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// from AlignAddr for it to have the same value as if 584de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// ValueType was loaded from Addr. 585de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// 586de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// Mask: Value to mask with the value loaded from AlignAddr to 587de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// include only the part that would've been loaded from Addr. 588de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// 589de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// Inv_Mask: The inverse of Mask. 590de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 591de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic PartwordMaskValues createMaskInstrs(IRBuilder<> &Builder, Instruction *I, 592de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Type *ValueType, Value *Addr, 593de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned WordSize) { 594de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar PartwordMaskValues Ret; 595de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 596f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar BasicBlock *BB = I->getParent(); 59737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Function *F = BB->getParent(); 598de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Module *M = I->getModule(); 599de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 60037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines LLVMContext &Ctx = F->getContext(); 601de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const DataLayout &DL = M->getDataLayout(); 602de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 603de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned ValueSize = DL.getTypeStoreSize(ValueType); 604de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 605de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert(ValueSize < WordSize); 606de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 607de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Ret.ValueType = ValueType; 608de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Ret.WordType = Type::getIntNTy(Ctx, WordSize * 8); 609de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 610de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Type *WordPtrType = 611de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Ret.WordType->getPointerTo(Addr->getType()->getPointerAddressSpace()); 612de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 613de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *AddrInt = Builder.CreatePtrToInt(Addr, DL.getIntPtrType(Ctx)); 614de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Ret.AlignedAddr = Builder.CreateIntToPtr( 615de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateAnd(AddrInt, ~(uint64_t)(WordSize - 1)), WordPtrType, 616de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar "AlignedAddr"); 617de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 618de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *PtrLSB = Builder.CreateAnd(AddrInt, WordSize - 1, "PtrLSB"); 619de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (DL.isLittleEndian()) { 620de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // turn bytes into bits 621de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Ret.ShiftAmt = Builder.CreateShl(PtrLSB, 3); 622de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else { 623de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // turn bytes into bits, and count from the other side. 624de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Ret.ShiftAmt = 625de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateShl(Builder.CreateXor(PtrLSB, WordSize - ValueSize), 3); 626de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 627de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 628de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Ret.ShiftAmt = Builder.CreateTrunc(Ret.ShiftAmt, Ret.WordType, "ShiftAmt"); 629de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Ret.Mask = Builder.CreateShl( 630de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ConstantInt::get(Ret.WordType, (1 << ValueSize * 8) - 1), Ret.ShiftAmt, 631de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar "Mask"); 632de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Ret.Inv_Mask = Builder.CreateNot(Ret.Mask, "Inv_Mask"); 633de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 634de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return Ret; 635de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 636de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 637de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// Emit IR to implement a masked version of a given atomicrmw 638de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// operation. (That is, only the bits under the Mask should be 639de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// affected by the operation) 640de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic Value *performMaskedAtomicOp(AtomicRMWInst::BinOp Op, 641de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar IRBuilder<> &Builder, Value *Loaded, 642de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *Shifted_Inc, Value *Inc, 643de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const PartwordMaskValues &PMV) { 644de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar switch (Op) { 645de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicRMWInst::Xchg: { 646de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask); 647de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *FinalVal = Builder.CreateOr(Loaded_MaskOut, Shifted_Inc); 648de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return FinalVal; 649de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 650de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicRMWInst::Or: 651de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicRMWInst::Xor: 652de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Or/Xor won't affect any other bits, so can just be done 653de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // directly. 654de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return performAtomicOp(Op, Builder, Loaded, Shifted_Inc); 655de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicRMWInst::Add: 656de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicRMWInst::Sub: 657de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicRMWInst::And: 658de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicRMWInst::Nand: { 659de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // The other arithmetic ops need to be masked into place. 660de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *NewVal = performAtomicOp(Op, Builder, Loaded, Shifted_Inc); 661de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *NewVal_Masked = Builder.CreateAnd(NewVal, PMV.Mask); 662de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask); 663de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *FinalVal = Builder.CreateOr(Loaded_MaskOut, NewVal_Masked); 664de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return FinalVal; 665de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 666de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicRMWInst::Max: 667de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicRMWInst::Min: 668de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicRMWInst::UMax: 669de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicRMWInst::UMin: { 670de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Finally, comparison ops will operate on the full value, so 671de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // truncate down to the original size, and expand out again after 672de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // doing the operation. 673de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *Loaded_Shiftdown = Builder.CreateTrunc( 674de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateLShr(Loaded, PMV.ShiftAmt), PMV.ValueType); 675de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *NewVal = performAtomicOp(Op, Builder, Loaded_Shiftdown, Inc); 676de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *NewVal_Shiftup = Builder.CreateShl( 677de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateZExt(NewVal, PMV.WordType), PMV.ShiftAmt); 678de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask); 679de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *FinalVal = Builder.CreateOr(Loaded_MaskOut, NewVal_Shiftup); 680de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return FinalVal; 681de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 682de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar default: 683de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar llvm_unreachable("Unknown atomic op"); 684de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 685de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 686de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 687de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// Expand a sub-word atomicrmw operation into an appropriate 688de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// word-sized operation. 689de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// 690de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// It will create an LL/SC or cmpxchg loop, as appropriate, the same 691de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// way as a typical atomicrmw expansion. The only difference here is 692de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// that the operation inside of the loop must operate only upon a 693de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// part of the value. 694de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarvoid AtomicExpand::expandPartwordAtomicRMW( 695de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AtomicRMWInst *AI, TargetLoweringBase::AtomicExpansionKind ExpansionKind) { 696de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 697de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert(ExpansionKind == TargetLoweringBase::AtomicExpansionKind::CmpXChg); 698de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 699de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AtomicOrdering MemOpOrder = AI->getOrdering(); 700de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 701de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar IRBuilder<> Builder(AI); 702de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 703de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar PartwordMaskValues PMV = 704de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(), 705de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar TLI->getMinCmpXchgSizeInBits() / 8); 706de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 707de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *ValOperand_Shifted = 708de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType), 709de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar PMV.ShiftAmt, "ValOperand_Shifted"); 710de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 711de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar auto PerformPartwordOp = [&](IRBuilder<> &Builder, Value *Loaded) { 712de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return performMaskedAtomicOp(AI->getOperation(), Builder, Loaded, 713de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ValOperand_Shifted, AI->getValOperand(), PMV); 714de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar }; 715de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 716de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // TODO: When we're ready to support LLSC conversions too, use 717de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // insertRMWLLSCLoop here for ExpansionKind==LLSC. 718de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *OldResult = 719de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar insertRMWCmpXchgLoop(Builder, PMV.WordType, PMV.AlignedAddr, MemOpOrder, 720de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar PerformPartwordOp, createCmpXchgInstFun); 721de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *FinalOldResult = Builder.CreateTrunc( 722de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateLShr(OldResult, PMV.ShiftAmt), PMV.ValueType); 723de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AI->replaceAllUsesWith(FinalOldResult); 724de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AI->eraseFromParent(); 725de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 726de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 727de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarvoid AtomicExpand::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) { 728de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // The basic idea here is that we're expanding a cmpxchg of a 729de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // smaller memory size up to a word-sized cmpxchg. To do this, we 730de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // need to add a retry-loop for strong cmpxchg, so that 731de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // modifications to other parts of the word don't cause a spurious 732de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // failure. 733de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 734de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // This generates code like the following: 735de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // [[Setup mask values PMV.*]] 736de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // %NewVal_Shifted = shl i32 %NewVal, %PMV.ShiftAmt 737de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // %Cmp_Shifted = shl i32 %Cmp, %PMV.ShiftAmt 738de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // %InitLoaded = load i32* %addr 739de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // %InitLoaded_MaskOut = and i32 %InitLoaded, %PMV.Inv_Mask 740de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // br partword.cmpxchg.loop 741de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // partword.cmpxchg.loop: 742de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // %Loaded_MaskOut = phi i32 [ %InitLoaded_MaskOut, %entry ], 743de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // [ %OldVal_MaskOut, %partword.cmpxchg.failure ] 744de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // %FullWord_NewVal = or i32 %Loaded_MaskOut, %NewVal_Shifted 745de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // %FullWord_Cmp = or i32 %Loaded_MaskOut, %Cmp_Shifted 746de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // %NewCI = cmpxchg i32* %PMV.AlignedAddr, i32 %FullWord_Cmp, 747de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // i32 %FullWord_NewVal success_ordering failure_ordering 748de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // %OldVal = extractvalue { i32, i1 } %NewCI, 0 749de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // %Success = extractvalue { i32, i1 } %NewCI, 1 750de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // br i1 %Success, label %partword.cmpxchg.end, 751de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // label %partword.cmpxchg.failure 752de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // partword.cmpxchg.failure: 753de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // %OldVal_MaskOut = and i32 %OldVal, %PMV.Inv_Mask 754de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // %ShouldContinue = icmp ne i32 %Loaded_MaskOut, %OldVal_MaskOut 755de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // br i1 %ShouldContinue, label %partword.cmpxchg.loop, 756de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // label %partword.cmpxchg.end 757de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // partword.cmpxchg.end: 758de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // %tmp1 = lshr i32 %OldVal, %PMV.ShiftAmt 759de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // %FinalOldVal = trunc i32 %tmp1 to i8 760de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // %tmp2 = insertvalue { i8, i1 } undef, i8 %FinalOldVal, 0 761de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // %Res = insertvalue { i8, i1 } %25, i1 %Success, 1 762de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 763de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *Addr = CI->getPointerOperand(); 764de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *Cmp = CI->getCompareOperand(); 765de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *NewVal = CI->getNewValOperand(); 766de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 767de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BasicBlock *BB = CI->getParent(); 768de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Function *F = BB->getParent(); 769de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar IRBuilder<> Builder(CI); 770de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar LLVMContext &Ctx = Builder.getContext(); 771de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 772de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const int WordSize = TLI->getMinCmpXchgSizeInBits() / 8; 773de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 774de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BasicBlock *EndBB = 775de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BB->splitBasicBlock(CI->getIterator(), "partword.cmpxchg.end"); 776de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar auto FailureBB = 777de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BasicBlock::Create(Ctx, "partword.cmpxchg.failure", F, EndBB); 778de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar auto LoopBB = BasicBlock::Create(Ctx, "partword.cmpxchg.loop", F, FailureBB); 779de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 780de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // The split call above "helpfully" added a branch at the end of BB 781de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // (to the wrong place). 782de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar std::prev(BB->end())->eraseFromParent(); 783de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.SetInsertPoint(BB); 784de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 785de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar PartwordMaskValues PMV = createMaskInstrs( 786de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder, CI, CI->getCompareOperand()->getType(), Addr, WordSize); 787de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 788de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Shift the incoming values over, into the right location in the word. 789de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *NewVal_Shifted = 790de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateShl(Builder.CreateZExt(NewVal, PMV.WordType), PMV.ShiftAmt); 791de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *Cmp_Shifted = 792de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateShl(Builder.CreateZExt(Cmp, PMV.WordType), PMV.ShiftAmt); 793de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 794de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Load the entire current word, and mask into place the expected and new 795de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // values 796de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar LoadInst *InitLoaded = Builder.CreateLoad(PMV.WordType, PMV.AlignedAddr); 797de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar InitLoaded->setVolatile(CI->isVolatile()); 798de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *InitLoaded_MaskOut = Builder.CreateAnd(InitLoaded, PMV.Inv_Mask); 799de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateBr(LoopBB); 800de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 801de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // partword.cmpxchg.loop: 802de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.SetInsertPoint(LoopBB); 803de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar PHINode *Loaded_MaskOut = Builder.CreatePHI(PMV.WordType, 2); 804de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Loaded_MaskOut->addIncoming(InitLoaded_MaskOut, BB); 805de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 806de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Mask/Or the expected and new values into place in the loaded word. 807de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *FullWord_NewVal = Builder.CreateOr(Loaded_MaskOut, NewVal_Shifted); 808de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *FullWord_Cmp = Builder.CreateOr(Loaded_MaskOut, Cmp_Shifted); 809de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AtomicCmpXchgInst *NewCI = Builder.CreateAtomicCmpXchg( 810de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar PMV.AlignedAddr, FullWord_Cmp, FullWord_NewVal, CI->getSuccessOrdering(), 811de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CI->getFailureOrdering(), CI->getSynchScope()); 812de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar NewCI->setVolatile(CI->isVolatile()); 813de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // When we're building a strong cmpxchg, we need a loop, so you 814de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // might think we could use a weak cmpxchg inside. But, using strong 815de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // allows the below comparison for ShouldContinue, and we're 816de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // expecting the underlying cmpxchg to be a machine instruction, 817de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // which is strong anyways. 818de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar NewCI->setWeak(CI->isWeak()); 819de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 820de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *OldVal = Builder.CreateExtractValue(NewCI, 0); 821de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *Success = Builder.CreateExtractValue(NewCI, 1); 822de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 823de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (CI->isWeak()) 824de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateBr(EndBB); 825de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar else 826de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateCondBr(Success, EndBB, FailureBB); 827de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 828de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // partword.cmpxchg.failure: 829de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.SetInsertPoint(FailureBB); 830de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Upon failure, verify that the masked-out part of the loaded value 831de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // has been modified. If it didn't, abort the cmpxchg, since the 832de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // masked-in part must've. 833de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *OldVal_MaskOut = Builder.CreateAnd(OldVal, PMV.Inv_Mask); 834de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *ShouldContinue = Builder.CreateICmpNE(Loaded_MaskOut, OldVal_MaskOut); 835de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateCondBr(ShouldContinue, LoopBB, EndBB); 836de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 837de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Add the second value to the phi from above 838de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Loaded_MaskOut->addIncoming(OldVal_MaskOut, FailureBB); 839de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 840de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // partword.cmpxchg.end: 841de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.SetInsertPoint(CI); 842de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 843de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *FinalOldVal = Builder.CreateTrunc( 844de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateLShr(OldVal, PMV.ShiftAmt), PMV.ValueType); 845de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *Res = UndefValue::get(CI->getType()); 846de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Res = Builder.CreateInsertValue(Res, FinalOldVal, 0); 847de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Res = Builder.CreateInsertValue(Res, Success, 1); 848de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 849de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CI->replaceAllUsesWith(Res); 850de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CI->eraseFromParent(); 851de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 852de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 853de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarvoid AtomicExpand::expandAtomicOpToLLSC( 854de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Instruction *I, Type *ResultType, Value *Addr, AtomicOrdering MemOpOrder, 855de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar function_ref<Value *(IRBuilder<> &, Value *)> PerformOp) { 856de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar IRBuilder<> Builder(I); 857de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *Loaded = 858de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar insertRMWLLSCLoop(Builder, ResultType, Addr, MemOpOrder, PerformOp); 859de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 860de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar I->replaceAllUsesWith(Loaded); 861de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar I->eraseFromParent(); 862de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 863de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 864de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarValue *AtomicExpand::insertRMWLLSCLoop( 865de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar IRBuilder<> &Builder, Type *ResultTy, Value *Addr, 866de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AtomicOrdering MemOpOrder, 867de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar function_ref<Value *(IRBuilder<> &, Value *)> PerformOp) { 868de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar LLVMContext &Ctx = Builder.getContext(); 869de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BasicBlock *BB = Builder.GetInsertBlock(); 870de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Function *F = BB->getParent(); 87137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 87237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Given: atomicrmw some_op iN* %addr, iN %incr ordering 87337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // 87437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // The standard expansion we produce is: 87537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // [...] 87637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // atomicrmw.start: 87737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // %loaded = @load.linked(%addr) 87837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // %new = some_op iN %loaded, %incr 87937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // %stored = @store_conditional(%new, %addr) 88037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // %try_again = icmp i32 ne %stored, 0 88137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // br i1 %try_again, label %loop, label %atomicrmw.end 88237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // atomicrmw.end: 88337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // [...] 884de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BasicBlock *ExitBB = 885de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BB->splitBasicBlock(Builder.GetInsertPoint(), "atomicrmw.end"); 88637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines BasicBlock *LoopBB = BasicBlock::Create(Ctx, "atomicrmw.start", F, ExitBB); 88737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 88837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // The split call above "helpfully" added a branch at the end of BB (to the 889de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // wrong place). 89037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines std::prev(BB->end())->eraseFromParent(); 89137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.SetInsertPoint(BB); 89237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.CreateBr(LoopBB); 89337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 89437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Start the main loop block now that we've taken care of the preliminaries. 89537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.SetInsertPoint(LoopBB); 89637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Loaded = TLI->emitLoadLinked(Builder, Addr, MemOpOrder); 89737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 898f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *NewVal = PerformOp(Builder, Loaded); 89937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 90037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *StoreSuccess = 90137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TLI->emitStoreConditional(Builder, NewVal, Addr, MemOpOrder); 90237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *TryAgain = Builder.CreateICmpNE( 90337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines StoreSuccess, ConstantInt::get(IntegerType::get(Ctx, 32), 0), "tryagain"); 90437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.CreateCondBr(TryAgain, LoopBB, ExitBB); 90537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 90637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.SetInsertPoint(ExitBB, ExitBB->begin()); 907de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return Loaded; 908de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 90937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 910de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// Convert an atomic cmpxchg of a non-integral type to an integer cmpxchg of 911de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// the equivalent bitwidth. We used to not support pointer cmpxchg in the 912de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// IR. As a migration step, we convert back to what use to be the standard 913de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// way to represent a pointer cmpxchg so that we can update backends one by 914de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar/// one. 915de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarAtomicCmpXchgInst *AtomicExpand::convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI) { 916de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar auto *M = CI->getModule(); 917de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Type *NewTy = getCorrespondingIntegerType(CI->getCompareOperand()->getType(), 918de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar M->getDataLayout()); 91937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 920de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar IRBuilder<> Builder(CI); 921de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 922de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *Addr = CI->getPointerOperand(); 923de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Type *PT = PointerType::get(NewTy, 924de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Addr->getType()->getPointerAddressSpace()); 925de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *NewAddr = Builder.CreateBitCast(Addr, PT); 926de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 927de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *NewCmp = Builder.CreatePtrToInt(CI->getCompareOperand(), NewTy); 928de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *NewNewVal = Builder.CreatePtrToInt(CI->getNewValOperand(), NewTy); 929de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 930de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 931de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar auto *NewCI = Builder.CreateAtomicCmpXchg(NewAddr, NewCmp, NewNewVal, 932de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CI->getSuccessOrdering(), 933de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CI->getFailureOrdering(), 934de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CI->getSynchScope()); 935de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar NewCI->setVolatile(CI->isVolatile()); 936de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar NewCI->setWeak(CI->isWeak()); 937de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar DEBUG(dbgs() << "Replaced " << *CI << " with " << *NewCI << "\n"); 938de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 939de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *OldVal = Builder.CreateExtractValue(NewCI, 0); 940de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *Succ = Builder.CreateExtractValue(NewCI, 1); 941de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 942de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar OldVal = Builder.CreateIntToPtr(OldVal, CI->getCompareOperand()->getType()); 943de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 944de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *Res = UndefValue::get(CI->getType()); 945de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Res = Builder.CreateInsertValue(Res, OldVal, 0); 946de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Res = Builder.CreateInsertValue(Res, Succ, 1); 947de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 948de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CI->replaceAllUsesWith(Res); 949de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CI->eraseFromParent(); 950de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return NewCI; 95137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 95237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 953de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 95437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesbool AtomicExpand::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) { 95537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AtomicOrdering SuccessOrder = CI->getSuccessOrdering(); 95637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AtomicOrdering FailureOrder = CI->getFailureOrdering(); 95737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Addr = CI->getPointerOperand(); 95837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines BasicBlock *BB = CI->getParent(); 95937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Function *F = BB->getParent(); 96037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines LLVMContext &Ctx = F->getContext(); 961de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // If shouldInsertFencesForAtomic() returns true, then the target does not 962de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // want to deal with memory orders, and emitLeading/TrailingFence should take 963de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // care of everything. Otherwise, emitLeading/TrailingFence are no-op and we 96437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // should preserve the ordering. 965de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool ShouldInsertFencesForAtomic = TLI->shouldInsertFencesForAtomic(CI); 96637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AtomicOrdering MemOpOrder = 967de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ShouldInsertFencesForAtomic ? AtomicOrdering::Monotonic : SuccessOrder; 968de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 969de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // In implementations which use a barrier to achieve release semantics, we can 970de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // delay emitting this barrier until we know a store is actually going to be 971de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // attempted. The cost of this delay is that we need 2 copies of the block 972de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // emitting the load-linked, affecting code size. 973de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // 974de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Ideally, this logic would be unconditional except for the minsize check 975de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // since in other cases the extra blocks naturally collapse down to the 976de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // minimal loop. Unfortunately, this puts too much stress on later 977de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // optimisations so we avoid emitting the extra logic in those cases too. 978de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool HasReleasedLoadBB = !CI->isWeak() && ShouldInsertFencesForAtomic && 979de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SuccessOrder != AtomicOrdering::Monotonic && 980de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SuccessOrder != AtomicOrdering::Acquire && 981de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar !F->optForMinSize(); 982de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 983de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // There's no overhead for sinking the release barrier in a weak cmpxchg, so 984de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // do it even on minsize. 985de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool UseUnconditionalReleaseBarrier = F->optForMinSize() && !CI->isWeak(); 98637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 98737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Given: cmpxchg some_op iN* %addr, iN %desired, iN %new success_ord fail_ord 98837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // 98937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // The full expansion we produce is: 99037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // [...] 99137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // cmpxchg.start: 992de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // %unreleasedload = @load.linked(%addr) 993de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // %should_store = icmp eq %unreleasedload, %desired 994de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // br i1 %should_store, label %cmpxchg.fencedstore, 995f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // label %cmpxchg.nostore 996de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // cmpxchg.releasingstore: 997de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // fence? 998de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // br label cmpxchg.trystore 99937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // cmpxchg.trystore: 1000de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // %loaded.trystore = phi [%unreleasedload, %releasingstore], 1001de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // [%releasedload, %cmpxchg.releasedload] 100237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // %stored = @store_conditional(%new, %addr) 100337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // %success = icmp eq i32 %stored, 0 1004de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // br i1 %success, label %cmpxchg.success, 1005de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // label %cmpxchg.releasedload/%cmpxchg.failure 1006de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // cmpxchg.releasedload: 1007de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // %releasedload = @load.linked(%addr) 1008de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // %should_store = icmp eq %releasedload, %desired 1009de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // br i1 %should_store, label %cmpxchg.trystore, 1010de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // label %cmpxchg.failure 101137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // cmpxchg.success: 101237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // fence? 101337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // br label %cmpxchg.end 1014f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // cmpxchg.nostore: 1015de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // %loaded.nostore = phi [%unreleasedload, %cmpxchg.start], 1016de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // [%releasedload, 1017de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // %cmpxchg.releasedload/%cmpxchg.trystore] 1018f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // @load_linked_fail_balance()? 1019f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // br label %cmpxchg.failure 102037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // cmpxchg.failure: 102137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // fence? 102237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // br label %cmpxchg.end 102337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // cmpxchg.end: 1024de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // %loaded = phi [%loaded.nostore, %cmpxchg.failure], 1025de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // [%loaded.trystore, %cmpxchg.trystore] 102637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // %success = phi i1 [true, %cmpxchg.success], [false, %cmpxchg.failure] 102737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // %restmp = insertvalue { iN, i1 } undef, iN %loaded, 0 102837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // %res = insertvalue { iN, i1 } %restmp, i1 %success, 1 102937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // [...] 1030f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar BasicBlock *ExitBB = BB->splitBasicBlock(CI->getIterator(), "cmpxchg.end"); 103137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines auto FailureBB = BasicBlock::Create(Ctx, "cmpxchg.failure", F, ExitBB); 1032f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar auto NoStoreBB = BasicBlock::Create(Ctx, "cmpxchg.nostore", F, FailureBB); 1033f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar auto SuccessBB = BasicBlock::Create(Ctx, "cmpxchg.success", F, NoStoreBB); 1034de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar auto ReleasedLoadBB = 1035de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BasicBlock::Create(Ctx, "cmpxchg.releasedload", F, SuccessBB); 1036de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar auto TryStoreBB = 1037de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BasicBlock::Create(Ctx, "cmpxchg.trystore", F, ReleasedLoadBB); 1038de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar auto ReleasingStoreBB = 1039de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BasicBlock::Create(Ctx, "cmpxchg.fencedstore", F, TryStoreBB); 1040de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar auto StartBB = BasicBlock::Create(Ctx, "cmpxchg.start", F, ReleasingStoreBB); 104137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 104237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // This grabs the DebugLoc from CI 104337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IRBuilder<> Builder(CI); 104437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 104537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // The split call above "helpfully" added a branch at the end of BB (to the 104637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // wrong place), but we might want a fence too. It's easiest to just remove 104737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // the branch entirely. 104837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines std::prev(BB->end())->eraseFromParent(); 104937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.SetInsertPoint(BB); 1050de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (ShouldInsertFencesForAtomic && UseUnconditionalReleaseBarrier) 1051de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar TLI->emitLeadingFence(Builder, SuccessOrder, /*IsStore=*/true, 1052de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar /*IsLoad=*/true); 1053de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateBr(StartBB); 105437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 105537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Start the main loop block now that we've taken care of the preliminaries. 1056de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.SetInsertPoint(StartBB); 1057de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *UnreleasedLoad = TLI->emitLoadLinked(Builder, Addr, MemOpOrder); 1058de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *ShouldStore = Builder.CreateICmpEQ( 1059de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar UnreleasedLoad, CI->getCompareOperand(), "should_store"); 106037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 1061f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // If the cmpxchg doesn't actually need any ordering when it fails, we can 106237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // jump straight past that fence instruction (if it exists). 1063de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateCondBr(ShouldStore, ReleasingStoreBB, NoStoreBB); 1064de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1065de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.SetInsertPoint(ReleasingStoreBB); 1066de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (ShouldInsertFencesForAtomic && !UseUnconditionalReleaseBarrier) 1067de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar TLI->emitLeadingFence(Builder, SuccessOrder, /*IsStore=*/true, 1068de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar /*IsLoad=*/true); 1069de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateBr(TryStoreBB); 107037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 107137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.SetInsertPoint(TryStoreBB); 107237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *StoreSuccess = TLI->emitStoreConditional( 107337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder, CI->getNewValOperand(), Addr, MemOpOrder); 107437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines StoreSuccess = Builder.CreateICmpEQ( 107537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines StoreSuccess, ConstantInt::get(Type::getInt32Ty(Ctx), 0), "success"); 1076de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BasicBlock *RetryBB = HasReleasedLoadBB ? ReleasedLoadBB : StartBB; 107737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.CreateCondBr(StoreSuccess, SuccessBB, 1078de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CI->isWeak() ? FailureBB : RetryBB); 1079de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1080de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.SetInsertPoint(ReleasedLoadBB); 1081de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *SecondLoad; 1082de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (HasReleasedLoadBB) { 1083de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SecondLoad = TLI->emitLoadLinked(Builder, Addr, MemOpOrder); 1084de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ShouldStore = Builder.CreateICmpEQ(SecondLoad, CI->getCompareOperand(), 1085de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar "should_store"); 1086de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1087de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // If the cmpxchg doesn't actually need any ordering when it fails, we can 1088de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // jump straight past that fence instruction (if it exists). 1089de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateCondBr(ShouldStore, TryStoreBB, NoStoreBB); 1090de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else 1091de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateUnreachable(); 1092de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1093de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Make sure later instructions don't get reordered with a fence if 1094de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // necessary. 109537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.SetInsertPoint(SuccessBB); 1096de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (ShouldInsertFencesForAtomic) 1097de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar TLI->emitTrailingFence(Builder, SuccessOrder, /*IsStore=*/true, 1098de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar /*IsLoad=*/true); 109937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.CreateBr(ExitBB); 110037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 1101f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Builder.SetInsertPoint(NoStoreBB); 1102f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // In the failing case, where we don't execute the store-conditional, the 1103f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // target might want to balance out the load-linked with a dedicated 1104f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // instruction (e.g., on ARM, clearing the exclusive monitor). 1105f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar TLI->emitAtomicCmpXchgNoStoreLLBalance(Builder); 1106f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Builder.CreateBr(FailureBB); 1107f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 110837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.SetInsertPoint(FailureBB); 1109de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (ShouldInsertFencesForAtomic) 1110de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar TLI->emitTrailingFence(Builder, FailureOrder, /*IsStore=*/true, 1111de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar /*IsLoad=*/true); 111237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.CreateBr(ExitBB); 111337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 111437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Finally, we have control-flow based knowledge of whether the cmpxchg 111537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // succeeded or not. We expose this to later passes by converting any 1116de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // subsequent "icmp eq/ne %loaded, %oldval" into a use of an appropriate 1117de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // PHI. 111837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Builder.SetInsertPoint(ExitBB, ExitBB->begin()); 111937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines PHINode *Success = Builder.CreatePHI(Type::getInt1Ty(Ctx), 2); 112037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Success->addIncoming(ConstantInt::getTrue(Ctx), SuccessBB); 112137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Success->addIncoming(ConstantInt::getFalse(Ctx), FailureBB); 112237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 1123de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Setup the builder so we can create any PHIs we need. 1124de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *Loaded; 1125de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!HasReleasedLoadBB) 1126de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Loaded = UnreleasedLoad; 1127de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar else { 1128de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.SetInsertPoint(TryStoreBB, TryStoreBB->begin()); 1129de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar PHINode *TryStoreLoaded = Builder.CreatePHI(UnreleasedLoad->getType(), 2); 1130de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar TryStoreLoaded->addIncoming(UnreleasedLoad, ReleasingStoreBB); 1131de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar TryStoreLoaded->addIncoming(SecondLoad, ReleasedLoadBB); 1132de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1133de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.SetInsertPoint(NoStoreBB, NoStoreBB->begin()); 1134de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar PHINode *NoStoreLoaded = Builder.CreatePHI(UnreleasedLoad->getType(), 2); 1135de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar NoStoreLoaded->addIncoming(UnreleasedLoad, StartBB); 1136de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar NoStoreLoaded->addIncoming(SecondLoad, ReleasedLoadBB); 1137de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1138de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.SetInsertPoint(ExitBB, ++ExitBB->begin()); 1139de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar PHINode *ExitLoaded = Builder.CreatePHI(UnreleasedLoad->getType(), 2); 1140de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ExitLoaded->addIncoming(TryStoreLoaded, SuccessBB); 1141de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ExitLoaded->addIncoming(NoStoreLoaded, FailureBB); 1142de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1143de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Loaded = ExitLoaded; 1144de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 1145de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 114637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Look for any users of the cmpxchg that are just comparing the loaded value 114737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // against the desired one, and replace them with the CFG-derived version. 114837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SmallVector<ExtractValueInst *, 2> PrunedInsts; 114937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines for (auto User : CI->users()) { 115037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ExtractValueInst *EV = dyn_cast<ExtractValueInst>(User); 115137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!EV) 115237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines continue; 115337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 115437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines assert(EV->getNumIndices() == 1 && EV->getIndices()[0] <= 1 && 115537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines "weird extraction from { iN, i1 }"); 115637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 115737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (EV->getIndices()[0] == 0) 115837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EV->replaceAllUsesWith(Loaded); 115937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines else 116037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EV->replaceAllUsesWith(Success); 116137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 116237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines PrunedInsts.push_back(EV); 116337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 116437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 116537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // We can remove the instructions now we're no longer iterating through them. 116637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines for (auto EV : PrunedInsts) 116737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines EV->eraseFromParent(); 116837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 116937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!CI->use_empty()) { 117037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Some use of the full struct return that we don't understand has happened, 117137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // so we've got to reconstruct it properly. 117237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Res; 117337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Res = Builder.CreateInsertValue(UndefValue::get(CI->getType()), Loaded, 0); 117437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Res = Builder.CreateInsertValue(Res, Success, 1); 117537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 117637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CI->replaceAllUsesWith(Res); 117737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 117837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 117937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CI->eraseFromParent(); 118037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 118137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 118237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 118337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesbool AtomicExpand::isIdempotentRMW(AtomicRMWInst* RMWI) { 118437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines auto C = dyn_cast<ConstantInt>(RMWI->getValOperand()); 118537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if(!C) 118637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 118737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 118837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AtomicRMWInst::BinOp Op = RMWI->getOperation(); 118937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines switch(Op) { 119037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::Add: 119137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::Sub: 119237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::Or: 119337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::Xor: 119437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return C->isZero(); 119537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case AtomicRMWInst::And: 119637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return C->isMinusOne(); 119737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // FIXME: we could also treat Min/Max/UMin/UMax by the INT_MIN/INT_MAX/... 119837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines default: 119937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 120037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 120137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 120237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 120337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesbool AtomicExpand::simplifyIdempotentRMW(AtomicRMWInst* RMWI) { 120437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (auto ResultingLoad = TLI->lowerIdempotentRMWIntoFencedLoad(RMWI)) { 1205f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar tryExpandAtomicLoad(ResultingLoad); 120637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 120737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 120837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 120937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 1210f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 1211de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarValue *AtomicExpand::insertRMWCmpXchgLoop( 1212de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar IRBuilder<> &Builder, Type *ResultTy, Value *Addr, 1213de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AtomicOrdering MemOpOrder, 1214de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar function_ref<Value *(IRBuilder<> &, Value *)> PerformOp, 1215de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CreateCmpXchgInstFun CreateCmpXchg) { 1216de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar LLVMContext &Ctx = Builder.getContext(); 1217de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BasicBlock *BB = Builder.GetInsertBlock(); 1218f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Function *F = BB->getParent(); 1219f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 1220f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Given: atomicrmw some_op iN* %addr, iN %incr ordering 1221f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // 1222f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // The standard expansion we produce is: 1223f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // [...] 1224f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // %init_loaded = load atomic iN* %addr 1225f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // br label %loop 1226f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // loop: 1227f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // %loaded = phi iN [ %init_loaded, %entry ], [ %new_loaded, %loop ] 1228f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // %new = some_op iN %loaded, %incr 1229f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // %pair = cmpxchg iN* %addr, iN %loaded, iN %new 1230f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // %new_loaded = extractvalue { iN, i1 } %pair, 0 1231f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // %success = extractvalue { iN, i1 } %pair, 1 1232f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // br i1 %success, label %atomicrmw.end, label %loop 1233f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // atomicrmw.end: 1234f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // [...] 1235de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BasicBlock *ExitBB = 1236de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar BB->splitBasicBlock(Builder.GetInsertPoint(), "atomicrmw.end"); 1237f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar BasicBlock *LoopBB = BasicBlock::Create(Ctx, "atomicrmw.start", F, ExitBB); 1238f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 1239f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // The split call above "helpfully" added a branch at the end of BB (to the 1240f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // wrong place), but we want a load. It's easiest to just remove 1241f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // the branch entirely. 1242f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar std::prev(BB->end())->eraseFromParent(); 1243f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Builder.SetInsertPoint(BB); 1244de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar LoadInst *InitLoaded = Builder.CreateLoad(ResultTy, Addr); 1245f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Atomics require at least natural alignment. 1246de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar InitLoaded->setAlignment(ResultTy->getPrimitiveSizeInBits() / 8); 1247f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Builder.CreateBr(LoopBB); 1248f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 1249f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar // Start the main loop block now that we've taken care of the preliminaries. 1250f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Builder.SetInsertPoint(LoopBB); 1251de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar PHINode *Loaded = Builder.CreatePHI(ResultTy, 2, "loaded"); 1252f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Loaded->addIncoming(InitLoaded, BB); 1253f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 1254de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *NewVal = PerformOp(Builder, Loaded); 1255f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 1256f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *NewLoaded = nullptr; 1257f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Value *Success = nullptr; 1258f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 1259de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CreateCmpXchg(Builder, Addr, Loaded, NewVal, 1260de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar MemOpOrder == AtomicOrdering::Unordered 1261de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ? AtomicOrdering::Monotonic 1262de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar : MemOpOrder, 1263f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Success, NewLoaded); 1264f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar assert(Success && NewLoaded); 1265f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 1266f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Loaded->addIncoming(NewLoaded, LoopBB); 1267f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 1268f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Builder.CreateCondBr(Success, ExitBB, LoopBB); 1269f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 1270f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar Builder.SetInsertPoint(ExitBB, ExitBB->begin()); 1271de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return NewLoaded; 1272de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 1273f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 1274de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// Note: This function is exposed externally by AtomicExpandUtils.h 1275de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarbool llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI, 1276de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CreateCmpXchgInstFun CreateCmpXchg) { 1277de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar IRBuilder<> Builder(AI); 1278de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *Loaded = AtomicExpand::insertRMWCmpXchgLoop( 1279de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder, AI->getType(), AI->getPointerOperand(), AI->getOrdering(), 1280de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar [&](IRBuilder<> &Builder, Value *Loaded) { 1281de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return performAtomicOp(AI->getOperation(), Builder, Loaded, 1282de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AI->getValOperand()); 1283de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar }, 1284de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CreateCmpXchg); 1285de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1286de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AI->replaceAllUsesWith(Loaded); 1287f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar AI->eraseFromParent(); 1288de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return true; 1289de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 1290f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar 1291de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// In order to use one of the sized library calls such as 1292de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// __atomic_fetch_add_4, the alignment must be sufficient, the size 1293de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// must be one of the potentially-specialized sizes, and the value 1294de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// type must actually exist in C on the target (otherwise, the 1295de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// function wouldn't actually be defined.) 1296de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic bool canUseSizedAtomicCall(unsigned Size, unsigned Align, 1297de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const DataLayout &DL) { 1298de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // TODO: "LargestSize" is an approximation for "largest type that 1299de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // you can express in C". It seems to be the case that int128 is 1300de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // supported on all 64-bit platforms, otherwise only up to 64-bit 1301de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // integers are supported. If we get this wrong, then we'll try to 1302de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // call a sized libcall that doesn't actually exist. There should 1303de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // really be some more reliable way in LLVM of determining integer 1304de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // sizes which are valid in the target's C ABI... 1305de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned LargestSize = DL.getLargestLegalIntTypeSizeInBits() >= 64 ? 16 : 8; 1306de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return Align >= Size && 1307de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar (Size == 1 || Size == 2 || Size == 4 || Size == 8 || Size == 16) && 1308de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Size <= LargestSize; 1309de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 1310de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1311de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarvoid AtomicExpand::expandAtomicLoadToLibcall(LoadInst *I) { 1312de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar static const RTLIB::Libcall Libcalls[6] = { 1313de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::ATOMIC_LOAD, RTLIB::ATOMIC_LOAD_1, RTLIB::ATOMIC_LOAD_2, 1314de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::ATOMIC_LOAD_4, RTLIB::ATOMIC_LOAD_8, RTLIB::ATOMIC_LOAD_16}; 1315de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned Size = getAtomicOpSize(I); 1316de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned Align = getAtomicOpAlign(I); 1317de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1318de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool expanded = expandAtomicOpToLibcall( 1319de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar I, Size, Align, I->getPointerOperand(), nullptr, nullptr, 1320de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar I->getOrdering(), AtomicOrdering::NotAtomic, Libcalls); 1321de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar (void)expanded; 1322de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert(expanded && "expandAtomicOpToLibcall shouldn't fail tor Load"); 1323de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 1324de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1325de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarvoid AtomicExpand::expandAtomicStoreToLibcall(StoreInst *I) { 1326de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar static const RTLIB::Libcall Libcalls[6] = { 1327de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::ATOMIC_STORE, RTLIB::ATOMIC_STORE_1, RTLIB::ATOMIC_STORE_2, 1328de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::ATOMIC_STORE_4, RTLIB::ATOMIC_STORE_8, RTLIB::ATOMIC_STORE_16}; 1329de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned Size = getAtomicOpSize(I); 1330de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned Align = getAtomicOpAlign(I); 1331de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1332de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool expanded = expandAtomicOpToLibcall( 1333de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar I, Size, Align, I->getPointerOperand(), I->getValueOperand(), nullptr, 1334de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar I->getOrdering(), AtomicOrdering::NotAtomic, Libcalls); 1335de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar (void)expanded; 1336de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert(expanded && "expandAtomicOpToLibcall shouldn't fail tor Store"); 1337de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 1338de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1339de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarvoid AtomicExpand::expandAtomicCASToLibcall(AtomicCmpXchgInst *I) { 1340de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar static const RTLIB::Libcall Libcalls[6] = { 1341de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::ATOMIC_COMPARE_EXCHANGE, RTLIB::ATOMIC_COMPARE_EXCHANGE_1, 1342de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::ATOMIC_COMPARE_EXCHANGE_2, RTLIB::ATOMIC_COMPARE_EXCHANGE_4, 1343de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::ATOMIC_COMPARE_EXCHANGE_8, RTLIB::ATOMIC_COMPARE_EXCHANGE_16}; 1344de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned Size = getAtomicOpSize(I); 1345de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned Align = getAtomicOpAlign(I); 1346de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1347de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool expanded = expandAtomicOpToLibcall( 1348de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar I, Size, Align, I->getPointerOperand(), I->getNewValOperand(), 1349de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar I->getCompareOperand(), I->getSuccessOrdering(), I->getFailureOrdering(), 1350de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Libcalls); 1351de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar (void)expanded; 1352de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert(expanded && "expandAtomicOpToLibcall shouldn't fail tor CAS"); 1353de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 1354de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1355de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic ArrayRef<RTLIB::Libcall> GetRMWLibcall(AtomicRMWInst::BinOp Op) { 1356de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar static const RTLIB::Libcall LibcallsXchg[6] = { 1357de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::ATOMIC_EXCHANGE, RTLIB::ATOMIC_EXCHANGE_1, 1358de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::ATOMIC_EXCHANGE_2, RTLIB::ATOMIC_EXCHANGE_4, 1359de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::ATOMIC_EXCHANGE_8, RTLIB::ATOMIC_EXCHANGE_16}; 1360de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar static const RTLIB::Libcall LibcallsAdd[6] = { 1361de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_ADD_1, 1362de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::ATOMIC_FETCH_ADD_2, RTLIB::ATOMIC_FETCH_ADD_4, 1363de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::ATOMIC_FETCH_ADD_8, RTLIB::ATOMIC_FETCH_ADD_16}; 1364de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar static const RTLIB::Libcall LibcallsSub[6] = { 1365de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_SUB_1, 1366de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::ATOMIC_FETCH_SUB_2, RTLIB::ATOMIC_FETCH_SUB_4, 1367de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::ATOMIC_FETCH_SUB_8, RTLIB::ATOMIC_FETCH_SUB_16}; 1368de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar static const RTLIB::Libcall LibcallsAnd[6] = { 1369de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_AND_1, 1370de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::ATOMIC_FETCH_AND_2, RTLIB::ATOMIC_FETCH_AND_4, 1371de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::ATOMIC_FETCH_AND_8, RTLIB::ATOMIC_FETCH_AND_16}; 1372de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar static const RTLIB::Libcall LibcallsOr[6] = { 1373de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_OR_1, 1374de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::ATOMIC_FETCH_OR_2, RTLIB::ATOMIC_FETCH_OR_4, 1375de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::ATOMIC_FETCH_OR_8, RTLIB::ATOMIC_FETCH_OR_16}; 1376de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar static const RTLIB::Libcall LibcallsXor[6] = { 1377de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_XOR_1, 1378de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::ATOMIC_FETCH_XOR_2, RTLIB::ATOMIC_FETCH_XOR_4, 1379de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::ATOMIC_FETCH_XOR_8, RTLIB::ATOMIC_FETCH_XOR_16}; 1380de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar static const RTLIB::Libcall LibcallsNand[6] = { 1381de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_NAND_1, 1382de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::ATOMIC_FETCH_NAND_2, RTLIB::ATOMIC_FETCH_NAND_4, 1383de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::ATOMIC_FETCH_NAND_8, RTLIB::ATOMIC_FETCH_NAND_16}; 1384de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1385de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar switch (Op) { 1386de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicRMWInst::BAD_BINOP: 1387de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar llvm_unreachable("Should not have BAD_BINOP."); 1388de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicRMWInst::Xchg: 1389de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return makeArrayRef(LibcallsXchg); 1390de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicRMWInst::Add: 1391de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return makeArrayRef(LibcallsAdd); 1392de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicRMWInst::Sub: 1393de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return makeArrayRef(LibcallsSub); 1394de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicRMWInst::And: 1395de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return makeArrayRef(LibcallsAnd); 1396de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicRMWInst::Or: 1397de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return makeArrayRef(LibcallsOr); 1398de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicRMWInst::Xor: 1399de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return makeArrayRef(LibcallsXor); 1400de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicRMWInst::Nand: 1401de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return makeArrayRef(LibcallsNand); 1402de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicRMWInst::Max: 1403de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicRMWInst::Min: 1404de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicRMWInst::UMax: 1405de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicRMWInst::UMin: 1406de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // No atomic libcalls are available for max/min/umax/umin. 1407de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return {}; 1408de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 1409de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar llvm_unreachable("Unexpected AtomicRMW operation."); 1410de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 1411de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1412de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarvoid AtomicExpand::expandAtomicRMWToLibcall(AtomicRMWInst *I) { 1413de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ArrayRef<RTLIB::Libcall> Libcalls = GetRMWLibcall(I->getOperation()); 1414de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1415de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned Size = getAtomicOpSize(I); 1416de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned Align = getAtomicOpAlign(I); 1417de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1418de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool Success = false; 1419de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!Libcalls.empty()) 1420de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Success = expandAtomicOpToLibcall( 1421de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar I, Size, Align, I->getPointerOperand(), I->getValOperand(), nullptr, 1422de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar I->getOrdering(), AtomicOrdering::NotAtomic, Libcalls); 1423de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1424de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // The expansion failed: either there were no libcalls at all for 1425de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // the operation (min/max), or there were only size-specialized 1426de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // libcalls (add/sub/etc) and we needed a generic. So, expand to a 1427de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // CAS libcall, via a CAS loop, instead. 1428de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!Success) { 1429de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar expandAtomicRMWToCmpXchg(I, [this](IRBuilder<> &Builder, Value *Addr, 1430de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *Loaded, Value *NewVal, 1431de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AtomicOrdering MemOpOrder, 1432de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *&Success, Value *&NewLoaded) { 1433de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Create the CAS instruction normally... 1434de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg( 1435de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Addr, Loaded, NewVal, MemOpOrder, 1436de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AtomicCmpXchgInst::getStrongestFailureOrdering(MemOpOrder)); 1437de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Success = Builder.CreateExtractValue(Pair, 1, "success"); 1438de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded"); 1439de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1440de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // ...and then expand the CAS into a libcall. 1441de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar expandAtomicCASToLibcall(Pair); 1442de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar }); 1443de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 1444de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 1445de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1446de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// A helper routine for the above expandAtomic*ToLibcall functions. 1447de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// 1448de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// 'Libcalls' contains an array of enum values for the particular 1449de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// ATOMIC libcalls to be emitted. All of the other arguments besides 1450de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// 'I' are extracted from the Instruction subclass by the 1451de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// caller. Depending on the particular call, some will be null. 1452de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarbool AtomicExpand::expandAtomicOpToLibcall( 1453de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Instruction *I, unsigned Size, unsigned Align, Value *PointerOperand, 1454de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *ValueOperand, Value *CASExpected, AtomicOrdering Ordering, 1455de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AtomicOrdering Ordering2, ArrayRef<RTLIB::Libcall> Libcalls) { 1456de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert(Libcalls.size() == 6); 1457de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1458de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar LLVMContext &Ctx = I->getContext(); 1459de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Module *M = I->getModule(); 1460de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const DataLayout &DL = M->getDataLayout(); 1461de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar IRBuilder<> Builder(I); 1462de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar IRBuilder<> AllocaBuilder(&I->getFunction()->getEntryBlock().front()); 1463de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1464de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool UseSizedLibcall = canUseSizedAtomicCall(Size, Align, DL); 1465de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Type *SizedIntTy = Type::getIntNTy(Ctx, Size * 8); 1466de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1467de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar unsigned AllocaAlignment = DL.getPrefTypeAlignment(SizedIntTy); 1468de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1469de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // TODO: the "order" argument type is "int", not int32. So 1470de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // getInt32Ty may be wrong if the arch uses e.g. 16-bit ints. 1471de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ConstantInt *SizeVal64 = ConstantInt::get(Type::getInt64Ty(Ctx), Size); 1472de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert(Ordering != AtomicOrdering::NotAtomic && "expect atomic MO"); 1473de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Constant *OrderingVal = 1474de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ConstantInt::get(Type::getInt32Ty(Ctx), (int)toCABI(Ordering)); 1475de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Constant *Ordering2Val = nullptr; 1476de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (CASExpected) { 1477de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar assert(Ordering2 != AtomicOrdering::NotAtomic && "expect atomic MO"); 1478de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Ordering2Val = 1479de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ConstantInt::get(Type::getInt32Ty(Ctx), (int)toCABI(Ordering2)); 1480de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 1481de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar bool HasResult = I->getType() != Type::getVoidTy(Ctx); 1482de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1483de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLIB::Libcall RTLibType; 1484de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (UseSizedLibcall) { 1485de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar switch (Size) { 1486de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case 1: RTLibType = Libcalls[1]; break; 1487de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case 2: RTLibType = Libcalls[2]; break; 1488de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case 4: RTLibType = Libcalls[3]; break; 1489de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case 8: RTLibType = Libcalls[4]; break; 1490de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case 16: RTLibType = Libcalls[5]; break; 1491de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 1492de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else if (Libcalls[0] != RTLIB::UNKNOWN_LIBCALL) { 1493de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar RTLibType = Libcalls[0]; 1494de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else { 1495de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Can't use sized function, and there's no generic for this 1496de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // operation, so give up. 1497de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return false; 1498de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 1499de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1500de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Build up the function call. There's two kinds. First, the sized 1501de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // variants. These calls are going to be one of the following (with 1502de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // N=1,2,4,8,16): 1503de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // iN __atomic_load_N(iN *ptr, int ordering) 1504de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // void __atomic_store_N(iN *ptr, iN val, int ordering) 1505de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // iN __atomic_{exchange|fetch_*}_N(iN *ptr, iN val, int ordering) 1506de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // bool __atomic_compare_exchange_N(iN *ptr, iN *expected, iN desired, 1507de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // int success_order, int failure_order) 1508de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // 1509de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Note that these functions can be used for non-integer atomic 1510de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // operations, the values just need to be bitcast to integers on the 1511de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // way in and out. 1512de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // 1513de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // And, then, the generic variants. They look like the following: 1514de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // void __atomic_load(size_t size, void *ptr, void *ret, int ordering) 1515de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // void __atomic_store(size_t size, void *ptr, void *val, int ordering) 1516de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // void __atomic_exchange(size_t size, void *ptr, void *val, void *ret, 1517de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // int ordering) 1518de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // bool __atomic_compare_exchange(size_t size, void *ptr, void *expected, 1519de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // void *desired, int success_order, 1520de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // int failure_order) 1521de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // 1522de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // The different signatures are built up depending on the 1523de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // 'UseSizedLibcall', 'CASExpected', 'ValueOperand', and 'HasResult' 1524de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // variables. 1525de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1526de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AllocaInst *AllocaCASExpected = nullptr; 1527de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *AllocaCASExpected_i8 = nullptr; 1528de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AllocaInst *AllocaValue = nullptr; 1529de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *AllocaValue_i8 = nullptr; 1530de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AllocaInst *AllocaResult = nullptr; 1531de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *AllocaResult_i8 = nullptr; 1532de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1533de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Type *ResultTy; 1534de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SmallVector<Value *, 6> Args; 1535de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AttributeSet Attr; 1536de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1537de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // 'size' argument. 1538de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!UseSizedLibcall) { 1539de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Note, getIntPtrType is assumed equivalent to size_t. 1540de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Args.push_back(ConstantInt::get(DL.getIntPtrType(Ctx), Size)); 1541de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 1542de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1543de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // 'ptr' argument. 1544de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *PtrVal = 1545de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateBitCast(PointerOperand, Type::getInt8PtrTy(Ctx)); 1546de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Args.push_back(PtrVal); 1547de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1548de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // 'expected' argument, if present. 1549de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (CASExpected) { 1550de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AllocaCASExpected = AllocaBuilder.CreateAlloca(CASExpected->getType()); 1551de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AllocaCASExpected->setAlignment(AllocaAlignment); 1552de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AllocaCASExpected_i8 = 1553de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateBitCast(AllocaCASExpected, Type::getInt8PtrTy(Ctx)); 1554de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateLifetimeStart(AllocaCASExpected_i8, SizeVal64); 1555de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateAlignedStore(CASExpected, AllocaCASExpected, AllocaAlignment); 1556de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Args.push_back(AllocaCASExpected_i8); 1557de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 1558de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1559de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // 'val' argument ('desired' for cas), if present. 1560de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (ValueOperand) { 1561de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (UseSizedLibcall) { 1562de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *IntValue = 1563de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateBitOrPointerCast(ValueOperand, SizedIntTy); 1564de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Args.push_back(IntValue); 1565de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else { 1566de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AllocaValue = AllocaBuilder.CreateAlloca(ValueOperand->getType()); 1567de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AllocaValue->setAlignment(AllocaAlignment); 1568de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AllocaValue_i8 = 1569de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateBitCast(AllocaValue, Type::getInt8PtrTy(Ctx)); 1570de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateLifetimeStart(AllocaValue_i8, SizeVal64); 1571de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateAlignedStore(ValueOperand, AllocaValue, AllocaAlignment); 1572de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Args.push_back(AllocaValue_i8); 1573de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 1574de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 1575de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1576de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // 'ret' argument. 1577de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!CASExpected && HasResult && !UseSizedLibcall) { 1578de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AllocaResult = AllocaBuilder.CreateAlloca(I->getType()); 1579de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AllocaResult->setAlignment(AllocaAlignment); 1580de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AllocaResult_i8 = 1581de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateBitCast(AllocaResult, Type::getInt8PtrTy(Ctx)); 1582de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateLifetimeStart(AllocaResult_i8, SizeVal64); 1583de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Args.push_back(AllocaResult_i8); 1584de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 1585de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1586de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // 'ordering' ('success_order' for cas) argument. 1587de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Args.push_back(OrderingVal); 1588de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1589de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // 'failure_order' argument, if present. 1590de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Ordering2Val) 1591de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Args.push_back(Ordering2Val); 1592de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1593de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Now, the return type. 1594de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (CASExpected) { 1595de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ResultTy = Type::getInt1Ty(Ctx); 1596de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Attr = Attr.addAttribute(Ctx, AttributeSet::ReturnIndex, Attribute::ZExt); 1597de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else if (HasResult && UseSizedLibcall) 1598de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ResultTy = SizedIntTy; 1599de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar else 1600de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ResultTy = Type::getVoidTy(Ctx); 1601de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1602de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Done with setting up arguments and return types, create the call: 1603de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar SmallVector<Type *, 6> ArgTys; 1604de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar for (Value *Arg : Args) 1605de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ArgTys.push_back(Arg->getType()); 1606de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar FunctionType *FnType = FunctionType::get(ResultTy, ArgTys, false); 1607de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Constant *LibcallFn = 1608de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar M->getOrInsertFunction(TLI->getLibcallName(RTLibType), FnType, Attr); 1609de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CallInst *Call = Builder.CreateCall(LibcallFn, Args); 1610de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Call->setAttributes(Attr); 1611de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *Result = Call; 1612de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1613de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // And then, extract the results... 1614de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (ValueOperand && !UseSizedLibcall) 1615de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateLifetimeEnd(AllocaValue_i8, SizeVal64); 1616de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 1617de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (CASExpected) { 1618de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // The final result from the CAS is {load of 'expected' alloca, bool result 1619de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // from call} 1620de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Type *FinalResultTy = I->getType(); 1621de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *V = UndefValue::get(FinalResultTy); 1622de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *ExpectedOut = 1623de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateAlignedLoad(AllocaCASExpected, AllocaAlignment); 1624de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateLifetimeEnd(AllocaCASExpected_i8, SizeVal64); 1625de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar V = Builder.CreateInsertValue(V, ExpectedOut, 0); 1626de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar V = Builder.CreateInsertValue(V, Result, 1); 1627de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar I->replaceAllUsesWith(V); 1628de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else if (HasResult) { 1629de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *V; 1630de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (UseSizedLibcall) 1631de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar V = Builder.CreateBitOrPointerCast(Result, I->getType()); 1632de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar else { 1633de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar V = Builder.CreateAlignedLoad(AllocaResult, AllocaAlignment); 1634de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Builder.CreateLifetimeEnd(AllocaResult_i8, SizeVal64); 1635de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 1636de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar I->replaceAllUsesWith(V); 1637de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 1638de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar I->eraseFromParent(); 1639f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar return true; 1640f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar} 1641