1fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall//===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===// 2fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// 3fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// The LLVM Compiler Infrastructure 4fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// 5fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// This file is distributed under the University of Illinois Open Source 6fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// License. See LICENSE.TXT for details. 7fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// 8fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall//===----------------------------------------------------------------------===// 9fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// 10fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// This file contains the code for emitting atomic operations. 11fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// 12fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall//===----------------------------------------------------------------------===// 13fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 14fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall#include "CodeGenFunction.h" 15fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall#include "CGCall.h" 16fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall#include "CodeGenModule.h" 17fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall#include "clang/AST/ASTContext.h" 188b54999a831bb195c08541ca995ef0505c96193fMark Lacey#include "clang/CodeGen/CGFunctionInfo.h" 19e469249726840b8baa2df008ca891e213cc5c661Ed Schouten#include "llvm/ADT/StringExtras.h" 20fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall#include "llvm/IR/DataLayout.h" 21fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall#include "llvm/IR/Intrinsics.h" 229eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall#include "llvm/IR/Operator.h" 23fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 24fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCallusing namespace clang; 25fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCallusing namespace CodeGen; 26fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 279eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallnamespace { 289eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall class AtomicInfo { 299eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CodeGenFunction &CGF; 309eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall QualType AtomicTy; 319eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall QualType ValueTy; 329eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall uint64_t AtomicSizeInBits; 339eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall uint64_t ValueSizeInBits; 349eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CharUnits AtomicAlign; 359eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CharUnits ValueAlign; 369eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CharUnits LValueAlign; 379eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall TypeEvaluationKind EvaluationKind; 389eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall bool UseLibcall; 399eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall public: 409eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) : CGF(CGF) { 419eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall assert(lvalue.isSimple()); 429eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 439eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AtomicTy = lvalue.getType(); 449eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall ValueTy = AtomicTy->castAs<AtomicType>()->getValueType(); 459eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall EvaluationKind = CGF.getEvaluationKind(ValueTy); 469eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 479eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall ASTContext &C = CGF.getContext(); 489eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 499eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall uint64_t valueAlignInBits; 50651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines std::tie(ValueSizeInBits, valueAlignInBits) = C.getTypeInfo(ValueTy); 519eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 529eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall uint64_t atomicAlignInBits; 53651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines std::tie(AtomicSizeInBits, atomicAlignInBits) = C.getTypeInfo(AtomicTy); 549eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 559eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall assert(ValueSizeInBits <= AtomicSizeInBits); 569eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall assert(valueAlignInBits <= atomicAlignInBits); 579eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 589eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AtomicAlign = C.toCharUnitsFromBits(atomicAlignInBits); 599eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall ValueAlign = C.toCharUnitsFromBits(valueAlignInBits); 609eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (lvalue.getAlignment().isZero()) 619eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall lvalue.setAlignment(AtomicAlign); 629eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 639eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall UseLibcall = 649eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall (AtomicSizeInBits > uint64_t(C.toBits(lvalue.getAlignment())) || 659eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AtomicSizeInBits > C.getTargetInfo().getMaxAtomicInlineWidth()); 669eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 679eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 689eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall QualType getAtomicType() const { return AtomicTy; } 699eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall QualType getValueType() const { return ValueTy; } 709eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CharUnits getAtomicAlignment() const { return AtomicAlign; } 719eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CharUnits getValueAlignment() const { return ValueAlign; } 729eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; } 739eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall uint64_t getValueSizeInBits() const { return AtomicSizeInBits; } 749eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; } 759eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall bool shouldUseLibcall() const { return UseLibcall; } 769eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 779eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// Is the atomic size larger than the underlying value type? 789eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// 799eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// Note that the absence of padding does not mean that atomic 809eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// objects are completely interchangeable with non-atomic 819eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// objects: we might have promoted the alignment of a type 829eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// without making it bigger. 839eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall bool hasPadding() const { 849eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return (ValueSizeInBits != AtomicSizeInBits); 859eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 869eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 87336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman bool emitMemSetZeroIfNecessary(LValue dest) const; 889eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 899eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *getAtomicSizeValue() const { 909eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits); 919eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return CGF.CGM.getSize(size); 929eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 939eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 949eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// Cast the given pointer to an integer pointer suitable for 959eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// atomic operations. 969eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *emitCastToAtomicIntPointer(llvm::Value *addr) const; 979eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 989eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// Turn an atomic-layout object into an r-value. 999eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall RValue convertTempToRValue(llvm::Value *addr, 1004ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky AggValueSlot resultSlot, 1014ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky SourceLocation loc) const; 1029eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1039eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// Copy an atomic r-value into atomic-layout memory. 1049eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall void emitCopyIntoMemory(RValue rvalue, LValue lvalue) const; 1059eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1069eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// Project an l-value down to the value field. 1079eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall LValue projectValue(LValue lvalue) const { 1089eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *addr = lvalue.getAddress(); 1099eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (hasPadding()) 1109eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall addr = CGF.Builder.CreateStructGEP(addr, 0); 1119eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1129eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return LValue::MakeAddr(addr, getValueType(), lvalue.getAlignment(), 1139eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CGF.getContext(), lvalue.getTBAAInfo()); 1149eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 1159eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1169eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// Materialize an atomic r-value in atomic-layout memory. 1179eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *materializeRValue(RValue rvalue) const; 1189eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1199eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall private: 1209eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall bool requiresMemSetZero(llvm::Type *type) const; 1219eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall }; 1229eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 1239eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1249eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallstatic RValue emitAtomicLibcall(CodeGenFunction &CGF, 1259eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall StringRef fnName, 1269eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall QualType resultType, 1279eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CallArgList &args) { 1289eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall const CGFunctionInfo &fnInfo = 1299eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CGF.CGM.getTypes().arrangeFreeFunctionCall(resultType, args, 1309eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall FunctionType::ExtInfo(), RequiredArgs::All); 1319eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo); 1329eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName); 1339eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return CGF.EmitCall(fnInfo, fn, ReturnValueSlot(), args); 1349eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 1359eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1369eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// Does a store of the given IR type modify the full expected width? 1379eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallstatic bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, 1389eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall uint64_t expectedSize) { 1399eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize); 1409eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 1419eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1429eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// Does the atomic type require memsetting to zero before initialization? 1439eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// 1449eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// The IR type is provided as a way of making certain queries faster. 1459eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallbool AtomicInfo::requiresMemSetZero(llvm::Type *type) const { 1469eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // If the atomic type has size padding, we definitely need a memset. 1479eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (hasPadding()) return true; 1489eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1499eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Otherwise, do some simple heuristics to try to avoid it: 1509eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall switch (getEvaluationKind()) { 1519eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // For scalars and complexes, check whether the store size of the 1529eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // type uses the full size. 1539eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall case TEK_Scalar: 1549eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits); 1559eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall case TEK_Complex: 1569eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return !isFullSizeType(CGF.CGM, type->getStructElementType(0), 1579eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AtomicSizeInBits / 2); 1589eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 159336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman // Padding in structs has an undefined bit pattern. User beware. 1609eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall case TEK_Aggregate: 161336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman return false; 1629eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 1639eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm_unreachable("bad evaluation kind"); 1649eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 1659eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 166336d9df5e628279425344d754dc68047fa5b00a7Eli Friedmanbool AtomicInfo::emitMemSetZeroIfNecessary(LValue dest) const { 1679eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *addr = dest.getAddress(); 1689eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (!requiresMemSetZero(addr->getType()->getPointerElementType())) 169336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman return false; 1709eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1719eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CGF.Builder.CreateMemSet(addr, llvm::ConstantInt::get(CGF.Int8Ty, 0), 1729eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AtomicSizeInBits / 8, 1739eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall dest.getAlignment().getQuantity()); 174336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman return true; 1759eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 1769eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 177ef8225444452a1486bd721f3285301fe84643b00Stephen Hinesstatic void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, 178651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Value *Dest, llvm::Value *Ptr, 179651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Value *Val1, llvm::Value *Val2, 180651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines uint64_t Size, unsigned Align, 181651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::AtomicOrdering SuccessOrder, 182651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::AtomicOrdering FailureOrder) { 183651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment. 184651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::LoadInst *Expected = CGF.Builder.CreateLoad(Val1); 185651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Expected->setAlignment(Align); 186651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::LoadInst *Desired = CGF.Builder.CreateLoad(Val2); 187651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Desired->setAlignment(Align); 188651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 189ef8225444452a1486bd721f3285301fe84643b00Stephen Hines llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg( 190651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Ptr, Expected, Desired, SuccessOrder, FailureOrder); 191ef8225444452a1486bd721f3285301fe84643b00Stephen Hines Pair->setVolatile(E->isVolatile()); 192ef8225444452a1486bd721f3285301fe84643b00Stephen Hines Pair->setWeak(IsWeak); 193651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 194651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Cmp holds the result of the compare-exchange operation: true on success, 195651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // false on failure. 196ef8225444452a1486bd721f3285301fe84643b00Stephen Hines llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0); 197ef8225444452a1486bd721f3285301fe84643b00Stephen Hines llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1); 198651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 199651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // This basic block is used to hold the store instruction if the operation 200651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // failed. 201651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::BasicBlock *StoreExpectedBB = 202651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn); 203651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 204651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // This basic block is the exit point of the operation, we should end up 205651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // here regardless of whether or not the operation succeeded. 206651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::BasicBlock *ContinueBB = 207651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 208651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 209651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Update Expected if Expected isn't equal to Old, otherwise branch to the 210651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // exit point. 211651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB); 212651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 213651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.SetInsertPoint(StoreExpectedBB); 214651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Update the memory at Expected with Old's value. 215651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::StoreInst *StoreExpected = CGF.Builder.CreateStore(Old, Val1); 216651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines StoreExpected->setAlignment(Align); 217651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Finally, branch to the exit point. 218651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.CreateBr(ContinueBB); 219651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 220651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.SetInsertPoint(ContinueBB); 221651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Update the memory at Dest with Cmp's value. 222651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType())); 223651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return; 224651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 225651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 226651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines/// Given an ordering required on success, emit all possible cmpxchg 227651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines/// instructions to cope with the provided (but possibly only dynamically known) 228651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines/// FailureOrder. 229651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, 230ef8225444452a1486bd721f3285301fe84643b00Stephen Hines bool IsWeak, llvm::Value *Dest, 231ef8225444452a1486bd721f3285301fe84643b00Stephen Hines llvm::Value *Ptr, llvm::Value *Val1, 232ef8225444452a1486bd721f3285301fe84643b00Stephen Hines llvm::Value *Val2, 233651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Value *FailureOrderVal, 234651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines uint64_t Size, unsigned Align, 235651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::AtomicOrdering SuccessOrder) { 236651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::AtomicOrdering FailureOrder; 237651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) { 238651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines switch (FO->getSExtValue()) { 239651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines default: 240651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines FailureOrder = llvm::Monotonic; 241651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 242651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case AtomicExpr::AO_ABI_memory_order_consume: 243651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case AtomicExpr::AO_ABI_memory_order_acquire: 244651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines FailureOrder = llvm::Acquire; 245651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 246651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case AtomicExpr::AO_ABI_memory_order_seq_cst: 247651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines FailureOrder = llvm::SequentiallyConsistent; 248651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines break; 249651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 250651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (FailureOrder >= SuccessOrder) { 251651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Don't assert on undefined behaviour. 252651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines FailureOrder = 253651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder); 254651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 255ef8225444452a1486bd721f3285301fe84643b00Stephen Hines emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, Align, 256ef8225444452a1486bd721f3285301fe84643b00Stephen Hines SuccessOrder, FailureOrder); 257651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return; 258651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 259651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 260651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Create all the relevant BB's 2616bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, 2626bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines *SeqCstBB = nullptr; 263651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn); 264651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (SuccessOrder != llvm::Monotonic && SuccessOrder != llvm::Release) 265651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn); 266651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (SuccessOrder == llvm::SequentiallyConsistent) 267651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn); 268651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 269651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn); 270651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 271651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB); 272651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 273651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Emit all the different atomics 274651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 275651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // MonotonicBB is arbitrarily chosen as the default case; in practice, this 276651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // doesn't matter unless someone is crazy enough to use something that 277651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // doesn't fold to a constant for the ordering. 278651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.SetInsertPoint(MonotonicBB); 279ef8225444452a1486bd721f3285301fe84643b00Stephen Hines emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, 280651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Size, Align, SuccessOrder, llvm::Monotonic); 281651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.CreateBr(ContBB); 282651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 283651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (AcquireBB) { 284651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.SetInsertPoint(AcquireBB); 285ef8225444452a1486bd721f3285301fe84643b00Stephen Hines emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, 286651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Size, Align, SuccessOrder, llvm::Acquire); 287651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.CreateBr(ContBB); 288651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume), 289651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AcquireBB); 290651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acquire), 291651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AcquireBB); 292651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 293651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (SeqCstBB) { 294651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.SetInsertPoint(SeqCstBB); 295ef8225444452a1486bd721f3285301fe84643b00Stephen Hines emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, 296651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Size, Align, SuccessOrder, llvm::SequentiallyConsistent); 297651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.CreateBr(ContBB); 298651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst), 299651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SeqCstBB); 300651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 301651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 302651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.SetInsertPoint(ContBB); 303651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 304651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 305651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest, 306651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Value *Ptr, llvm::Value *Val1, llvm::Value *Val2, 307ef8225444452a1486bd721f3285301fe84643b00Stephen Hines llvm::Value *IsWeak, llvm::Value *FailureOrder, 308ef8225444452a1486bd721f3285301fe84643b00Stephen Hines uint64_t Size, unsigned Align, 309ef8225444452a1486bd721f3285301fe84643b00Stephen Hines llvm::AtomicOrdering Order) { 310fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add; 311fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; 312fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 313fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall switch (E->getOp()) { 314fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_init: 315fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm_unreachable("Already handled!"); 316fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 317fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 318ef8225444452a1486bd721f3285301fe84643b00Stephen Hines emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 319ef8225444452a1486bd721f3285301fe84643b00Stephen Hines FailureOrder, Size, Align, Order); 320ef8225444452a1486bd721f3285301fe84643b00Stephen Hines return; 321fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 322ef8225444452a1486bd721f3285301fe84643b00Stephen Hines emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 323ef8225444452a1486bd721f3285301fe84643b00Stephen Hines FailureOrder, Size, Align, Order); 324ef8225444452a1486bd721f3285301fe84643b00Stephen Hines return; 325fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_compare_exchange: 326ef8225444452a1486bd721f3285301fe84643b00Stephen Hines case AtomicExpr::AO__atomic_compare_exchange_n: { 327ef8225444452a1486bd721f3285301fe84643b00Stephen Hines if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) { 328ef8225444452a1486bd721f3285301fe84643b00Stephen Hines emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr, 329ef8225444452a1486bd721f3285301fe84643b00Stephen Hines Val1, Val2, FailureOrder, Size, Align, Order); 330ef8225444452a1486bd721f3285301fe84643b00Stephen Hines } else { 331ef8225444452a1486bd721f3285301fe84643b00Stephen Hines // Create all the relevant BB's 332ef8225444452a1486bd721f3285301fe84643b00Stephen Hines llvm::BasicBlock *StrongBB = 333ef8225444452a1486bd721f3285301fe84643b00Stephen Hines CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn); 334ef8225444452a1486bd721f3285301fe84643b00Stephen Hines llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn); 335ef8225444452a1486bd721f3285301fe84643b00Stephen Hines llvm::BasicBlock *ContBB = 336ef8225444452a1486bd721f3285301fe84643b00Stephen Hines CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 337ef8225444452a1486bd721f3285301fe84643b00Stephen Hines 338ef8225444452a1486bd721f3285301fe84643b00Stephen Hines llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB); 339ef8225444452a1486bd721f3285301fe84643b00Stephen Hines SI->addCase(CGF.Builder.getInt1(false), StrongBB); 340ef8225444452a1486bd721f3285301fe84643b00Stephen Hines 341ef8225444452a1486bd721f3285301fe84643b00Stephen Hines CGF.Builder.SetInsertPoint(StrongBB); 342ef8225444452a1486bd721f3285301fe84643b00Stephen Hines emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 343ef8225444452a1486bd721f3285301fe84643b00Stephen Hines FailureOrder, Size, Align, Order); 344ef8225444452a1486bd721f3285301fe84643b00Stephen Hines CGF.Builder.CreateBr(ContBB); 345ef8225444452a1486bd721f3285301fe84643b00Stephen Hines 346ef8225444452a1486bd721f3285301fe84643b00Stephen Hines CGF.Builder.SetInsertPoint(WeakBB); 347ef8225444452a1486bd721f3285301fe84643b00Stephen Hines emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 348ef8225444452a1486bd721f3285301fe84643b00Stephen Hines FailureOrder, Size, Align, Order); 349ef8225444452a1486bd721f3285301fe84643b00Stephen Hines CGF.Builder.CreateBr(ContBB); 350ef8225444452a1486bd721f3285301fe84643b00Stephen Hines 351ef8225444452a1486bd721f3285301fe84643b00Stephen Hines CGF.Builder.SetInsertPoint(ContBB); 352ef8225444452a1486bd721f3285301fe84643b00Stephen Hines } 353fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall return; 354ef8225444452a1486bd721f3285301fe84643b00Stephen Hines } 355fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_load: 356fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_load_n: 357fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_load: { 358fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr); 359fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Load->setAtomic(Order); 360fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Load->setAlignment(Size); 361fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Load->setVolatile(E->isVolatile()); 362fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Load, Dest); 363fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall StoreDest->setAlignment(Align); 364fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall return; 365fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 366fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 367fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_store: 368fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_store: 369fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_store_n: { 370fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall assert(!Dest && "Store does not return a value"); 371fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1); 372fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall LoadVal1->setAlignment(Align); 373fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr); 374fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Store->setAtomic(Order); 375fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Store->setAlignment(Size); 376fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Store->setVolatile(E->isVolatile()); 377fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall return; 378fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 379fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 380fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_exchange: 381fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_exchange_n: 382fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_exchange: 383fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Op = llvm::AtomicRMWInst::Xchg; 384fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 385fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 386fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_add_fetch: 387fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall PostOp = llvm::Instruction::Add; 388fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Fall through. 389fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_add: 390fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_add: 391fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Op = llvm::AtomicRMWInst::Add; 392fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 393fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 394fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_sub_fetch: 395fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall PostOp = llvm::Instruction::Sub; 396fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Fall through. 397fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_sub: 398fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_sub: 399fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Op = llvm::AtomicRMWInst::Sub; 400fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 401fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 402fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_and_fetch: 403fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall PostOp = llvm::Instruction::And; 404fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Fall through. 405fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_and: 406fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_and: 407fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Op = llvm::AtomicRMWInst::And; 408fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 409fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 410fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_or_fetch: 411fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall PostOp = llvm::Instruction::Or; 412fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Fall through. 413fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_or: 414fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_or: 415fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Op = llvm::AtomicRMWInst::Or; 416fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 417fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 418fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_xor_fetch: 419fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall PostOp = llvm::Instruction::Xor; 420fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Fall through. 421fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_xor: 422fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_xor: 423fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Op = llvm::AtomicRMWInst::Xor; 424fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 425fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 426fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_nand_fetch: 427fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall PostOp = llvm::Instruction::And; 428fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Fall through. 429fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_nand: 430fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Op = llvm::AtomicRMWInst::Nand; 431fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 432fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 433fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 434fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1); 435fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall LoadVal1->setAlignment(Align); 436fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::AtomicRMWInst *RMWI = 437fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall CGF.Builder.CreateAtomicRMW(Op, Ptr, LoadVal1, Order); 438fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall RMWI->setVolatile(E->isVolatile()); 439fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 440fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // For __atomic_*_fetch operations, perform the operation again to 441fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // determine the value which was written. 442fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::Value *Result = RMWI; 443fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (PostOp) 444fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1); 445fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) 446fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Result = CGF.Builder.CreateNot(Result); 447fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Result, Dest); 448fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall StoreDest->setAlignment(Align); 449fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall} 450fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 451fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// This function emits any expression (scalar, complex, or aggregate) 452fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// into a temporary alloca. 453fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCallstatic llvm::Value * 454fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCallEmitValToTemp(CodeGenFunction &CGF, Expr *E) { 455fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::Value *DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp"); 456fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(), 457fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall /*Init*/ true); 458fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall return DeclPtr; 459fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall} 460fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 461e469249726840b8baa2df008ca891e213cc5c661Ed Schoutenstatic void 462e469249726840b8baa2df008ca891e213cc5c661Ed SchoutenAddDirectArgument(CodeGenFunction &CGF, CallArgList &Args, 4634ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy, 4644ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky SourceLocation Loc) { 465e469249726840b8baa2df008ca891e213cc5c661Ed Schouten if (UseOptimizedLibcall) { 466e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // Load value and pass it to the function directly. 467e469249726840b8baa2df008ca891e213cc5c661Ed Schouten unsigned Align = CGF.getContext().getTypeAlignInChars(ValTy).getQuantity(); 4684ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky Val = CGF.EmitLoadOfScalar(Val, false, Align, ValTy, Loc); 469e469249726840b8baa2df008ca891e213cc5c661Ed Schouten Args.add(RValue::get(Val), ValTy); 470e469249726840b8baa2df008ca891e213cc5c661Ed Schouten } else { 471e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // Non-optimized functions always take a reference. 472e469249726840b8baa2df008ca891e213cc5c661Ed Schouten Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)), 473e469249726840b8baa2df008ca891e213cc5c661Ed Schouten CGF.getContext().VoidPtrTy); 474e469249726840b8baa2df008ca891e213cc5c661Ed Schouten } 475e469249726840b8baa2df008ca891e213cc5c661Ed Schouten} 476e469249726840b8baa2df008ca891e213cc5c661Ed Schouten 477fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCallRValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) { 478fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall QualType AtomicTy = E->getPtr()->getType()->getPointeeType(); 479fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall QualType MemTy = AtomicTy; 480fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (const AtomicType *AT = AtomicTy->getAs<AtomicType>()) 481fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall MemTy = AT->getValueType(); 482fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall CharUnits sizeChars = getContext().getTypeSizeInChars(AtomicTy); 483fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall uint64_t Size = sizeChars.getQuantity(); 484fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall CharUnits alignChars = getContext().getTypeAlignInChars(AtomicTy); 485fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall unsigned Align = alignChars.getQuantity(); 486fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall unsigned MaxInlineWidthInBits = 48764aa4b3ec7e62288e2e66c1935487ece995ca94bJohn McCall getTarget().getMaxAtomicInlineWidth(); 488fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall bool UseLibcall = (Size != Align || 489fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall getContext().toBits(sizeChars) > MaxInlineWidthInBits); 490fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 491ef8225444452a1486bd721f3285301fe84643b00Stephen Hines llvm::Value *IsWeak = nullptr, *OrderFail = nullptr, *Val1 = nullptr, 492ef8225444452a1486bd721f3285301fe84643b00Stephen Hines *Val2 = nullptr; 4936bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines llvm::Value *Ptr = EmitScalarExpr(E->getPtr()); 494fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 495fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (E->getOp() == AtomicExpr::AO__c11_atomic_init) { 496fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall assert(!Dest && "Init does not return a value"); 4979eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall LValue lvalue = LValue::MakeAddr(Ptr, AtomicTy, alignChars, getContext()); 4989eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall EmitAtomicInit(E->getVal1(), lvalue); 4996bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 500fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 501fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 5026bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines llvm::Value *Order = EmitScalarExpr(E->getOrder()); 503fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 504fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall switch (E->getOp()) { 505fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_init: 506fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm_unreachable("Already handled!"); 507fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 508fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_load: 509fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_load_n: 510fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 511fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 512fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_load: 513fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Dest = EmitScalarExpr(E->getVal1()); 514fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 515fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 516fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_store: 517fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Val1 = EmitScalarExpr(E->getVal1()); 518fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 519fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 520fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_exchange: 521fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Val1 = EmitScalarExpr(E->getVal1()); 522fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Dest = EmitScalarExpr(E->getVal2()); 523fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 524fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 525fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 526fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 527fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_compare_exchange_n: 528fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_compare_exchange: 529fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Val1 = EmitScalarExpr(E->getVal1()); 530fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange) 531fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Val2 = EmitScalarExpr(E->getVal2()); 532fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall else 533fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Val2 = EmitValToTemp(*this, E->getVal2()); 534fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall OrderFail = EmitScalarExpr(E->getOrderFail()); 535fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (E->getNumSubExprs() == 6) 536ef8225444452a1486bd721f3285301fe84643b00Stephen Hines IsWeak = EmitScalarExpr(E->getWeak()); 537fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 538fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 539fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_add: 540fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_sub: 541fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (MemTy->isPointerType()) { 542fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // For pointer arithmetic, we're required to do a bit of math: 543fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // adding 1 to an int* is not the same as adding 1 to a uintptr_t. 544fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // ... but only for the C11 builtins. The GNU builtins expect the 545fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // user to multiply by sizeof(T). 546fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall QualType Val1Ty = E->getVal1()->getType(); 547fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1()); 548fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall CharUnits PointeeIncAmt = 549fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall getContext().getTypeSizeInChars(MemTy->getPointeeType()); 550fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt)); 551fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Val1 = CreateMemTemp(Val1Ty, ".atomictmp"); 552fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Val1, Val1Ty)); 553fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 554fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 555fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Fall through. 556fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_add: 557fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_sub: 558fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_add_fetch: 559fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_sub_fetch: 560fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_store: 561fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_exchange: 562fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_store_n: 563fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_exchange_n: 564fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_and: 565fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_or: 566fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_xor: 567fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_and: 568fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_or: 569fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_xor: 570fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_nand: 571fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_and_fetch: 572fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_or_fetch: 573fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_xor_fetch: 574fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_nand_fetch: 575fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Val1 = EmitValToTemp(*this, E->getVal1()); 576fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 577fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 578fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 579fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (!E->getType()->isVoidType() && !Dest) 580fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Dest = CreateMemTemp(E->getType(), ".atomicdst"); 581fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 582fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary . 583fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (UseLibcall) { 584e469249726840b8baa2df008ca891e213cc5c661Ed Schouten bool UseOptimizedLibcall = false; 585e469249726840b8baa2df008ca891e213cc5c661Ed Schouten switch (E->getOp()) { 586e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__c11_atomic_fetch_add: 587e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__atomic_fetch_add: 588e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__c11_atomic_fetch_and: 589e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__atomic_fetch_and: 590e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__c11_atomic_fetch_or: 591e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__atomic_fetch_or: 592e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__c11_atomic_fetch_sub: 593e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__atomic_fetch_sub: 594e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__c11_atomic_fetch_xor: 595e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__atomic_fetch_xor: 596e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // For these, only library calls for certain sizes exist. 597e469249726840b8baa2df008ca891e213cc5c661Ed Schouten UseOptimizedLibcall = true; 598e469249726840b8baa2df008ca891e213cc5c661Ed Schouten break; 599e469249726840b8baa2df008ca891e213cc5c661Ed Schouten default: 600e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // Only use optimized library calls for sizes for which they exist. 601e469249726840b8baa2df008ca891e213cc5c661Ed Schouten if (Size == 1 || Size == 2 || Size == 4 || Size == 8) 602e469249726840b8baa2df008ca891e213cc5c661Ed Schouten UseOptimizedLibcall = true; 603e469249726840b8baa2df008ca891e213cc5c661Ed Schouten break; 604e469249726840b8baa2df008ca891e213cc5c661Ed Schouten } 605fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 606fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall CallArgList Args; 607e469249726840b8baa2df008ca891e213cc5c661Ed Schouten if (!UseOptimizedLibcall) { 608e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // For non-optimized library calls, the size is the first parameter 609e469249726840b8baa2df008ca891e213cc5c661Ed Schouten Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)), 610e469249726840b8baa2df008ca891e213cc5c661Ed Schouten getContext().getSizeType()); 611e469249726840b8baa2df008ca891e213cc5c661Ed Schouten } 612e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // Atomic address is the first or second parameter 6135d4a7559df106959dd721744c8971547f0f09097Nick Lewycky Args.add(RValue::get(EmitCastToVoidPtr(Ptr)), getContext().VoidPtrTy); 614fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 615e469249726840b8baa2df008ca891e213cc5c661Ed Schouten std::string LibCallName; 616651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines QualType LoweredMemTy = 617651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy; 618e469249726840b8baa2df008ca891e213cc5c661Ed Schouten QualType RetTy; 619e469249726840b8baa2df008ca891e213cc5c661Ed Schouten bool HaveRetTy = false; 620fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall switch (E->getOp()) { 621fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // There is only one libcall for compare an exchange, because there is no 622fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // optimisation benefit possible from a libcall version of a weak compare 623fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // and exchange. 624e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // bool __atomic_compare_exchange(size_t size, void *mem, void *expected, 625fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // void *desired, int success, int failure) 626e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired, 627e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // int success, int failure) 628fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 629fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 630fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_compare_exchange: 631fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_compare_exchange_n: 632fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall LibCallName = "__atomic_compare_exchange"; 633fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall RetTy = getContext().BoolTy; 634e469249726840b8baa2df008ca891e213cc5c661Ed Schouten HaveRetTy = true; 6354ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky Args.add(RValue::get(EmitCastToVoidPtr(Val1)), getContext().VoidPtrTy); 6364ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2, MemTy, 6374ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky E->getExprLoc()); 6385d4a7559df106959dd721744c8971547f0f09097Nick Lewycky Args.add(RValue::get(Order), getContext().IntTy); 639fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Order = OrderFail; 640fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 641fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // void __atomic_exchange(size_t size, void *mem, void *val, void *return, 642fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // int order) 643e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // T __atomic_exchange_N(T *mem, T val, int order) 644fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_exchange: 645fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_exchange_n: 646fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_exchange: 647fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall LibCallName = "__atomic_exchange"; 6484ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy, 6494ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky E->getExprLoc()); 650fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 651fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // void __atomic_store(size_t size, void *mem, void *val, int order) 652e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // void __atomic_store_N(T *mem, T val, int order) 653fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_store: 654fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_store: 655fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_store_n: 656fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall LibCallName = "__atomic_store"; 657e469249726840b8baa2df008ca891e213cc5c661Ed Schouten RetTy = getContext().VoidTy; 658e469249726840b8baa2df008ca891e213cc5c661Ed Schouten HaveRetTy = true; 6594ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy, 6604ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky E->getExprLoc()); 661fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 662fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // void __atomic_load(size_t size, void *mem, void *return, int order) 663e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // T __atomic_load_N(T *mem, int order) 664fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_load: 665fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_load: 666fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_load_n: 667fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall LibCallName = "__atomic_load"; 668e469249726840b8baa2df008ca891e213cc5c661Ed Schouten break; 669e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // T __atomic_fetch_add_N(T *mem, T val, int order) 670e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__c11_atomic_fetch_add: 671e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__atomic_fetch_add: 672e469249726840b8baa2df008ca891e213cc5c661Ed Schouten LibCallName = "__atomic_fetch_add"; 673651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy, 6744ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky E->getExprLoc()); 675e469249726840b8baa2df008ca891e213cc5c661Ed Schouten break; 676e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // T __atomic_fetch_and_N(T *mem, T val, int order) 677e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__c11_atomic_fetch_and: 678e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__atomic_fetch_and: 679e469249726840b8baa2df008ca891e213cc5c661Ed Schouten LibCallName = "__atomic_fetch_and"; 6804ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy, 6814ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky E->getExprLoc()); 682e469249726840b8baa2df008ca891e213cc5c661Ed Schouten break; 683e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // T __atomic_fetch_or_N(T *mem, T val, int order) 684e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__c11_atomic_fetch_or: 685e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__atomic_fetch_or: 686e469249726840b8baa2df008ca891e213cc5c661Ed Schouten LibCallName = "__atomic_fetch_or"; 6874ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy, 6884ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky E->getExprLoc()); 689e469249726840b8baa2df008ca891e213cc5c661Ed Schouten break; 690e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // T __atomic_fetch_sub_N(T *mem, T val, int order) 691e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__c11_atomic_fetch_sub: 692e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__atomic_fetch_sub: 693e469249726840b8baa2df008ca891e213cc5c661Ed Schouten LibCallName = "__atomic_fetch_sub"; 694651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy, 6954ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky E->getExprLoc()); 696e469249726840b8baa2df008ca891e213cc5c661Ed Schouten break; 697e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // T __atomic_fetch_xor_N(T *mem, T val, int order) 698e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__c11_atomic_fetch_xor: 699e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__atomic_fetch_xor: 700e469249726840b8baa2df008ca891e213cc5c661Ed Schouten LibCallName = "__atomic_fetch_xor"; 7014ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy, 7024ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky E->getExprLoc()); 703fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 704fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall default: return EmitUnsupportedRValue(E, "atomic library call"); 705fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 706e469249726840b8baa2df008ca891e213cc5c661Ed Schouten 707e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // Optimized functions have the size in their name. 708e469249726840b8baa2df008ca891e213cc5c661Ed Schouten if (UseOptimizedLibcall) 709e469249726840b8baa2df008ca891e213cc5c661Ed Schouten LibCallName += "_" + llvm::utostr(Size); 710e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // By default, assume we return a value of the atomic type. 711e469249726840b8baa2df008ca891e213cc5c661Ed Schouten if (!HaveRetTy) { 712e469249726840b8baa2df008ca891e213cc5c661Ed Schouten if (UseOptimizedLibcall) { 713e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // Value is returned directly. 714e469249726840b8baa2df008ca891e213cc5c661Ed Schouten RetTy = MemTy; 715e469249726840b8baa2df008ca891e213cc5c661Ed Schouten } else { 716e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // Value is returned through parameter before the order. 717e469249726840b8baa2df008ca891e213cc5c661Ed Schouten RetTy = getContext().VoidTy; 718e469249726840b8baa2df008ca891e213cc5c661Ed Schouten Args.add(RValue::get(EmitCastToVoidPtr(Dest)), 719e469249726840b8baa2df008ca891e213cc5c661Ed Schouten getContext().VoidPtrTy); 720e469249726840b8baa2df008ca891e213cc5c661Ed Schouten } 721e469249726840b8baa2df008ca891e213cc5c661Ed Schouten } 722fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // order is always the last parameter 723fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Args.add(RValue::get(Order), 724fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall getContext().IntTy); 725fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 726fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall const CGFunctionInfo &FuncInfo = 727fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall CGM.getTypes().arrangeFreeFunctionCall(RetTy, Args, 728fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall FunctionType::ExtInfo(), RequiredArgs::All); 729fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo); 730fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName); 731fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall RValue Res = EmitCall(FuncInfo, Func, ReturnValueSlot(), Args); 732e469249726840b8baa2df008ca891e213cc5c661Ed Schouten if (!RetTy->isVoidType()) 733fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall return Res; 734fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (E->getType()->isVoidType()) 7356bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 7364ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky return convertTempToRValue(Dest, E->getType(), E->getExprLoc()); 737fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 738fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 739fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store || 740fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall E->getOp() == AtomicExpr::AO__atomic_store || 741fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall E->getOp() == AtomicExpr::AO__atomic_store_n; 742fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load || 743fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall E->getOp() == AtomicExpr::AO__atomic_load || 744fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall E->getOp() == AtomicExpr::AO__atomic_load_n; 745fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 746fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::Type *IPtrTy = 747fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::IntegerType::get(getLLVMContext(), Size * 8)->getPointerTo(); 748fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::Value *OrigDest = Dest; 749fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Ptr = Builder.CreateBitCast(Ptr, IPtrTy); 750fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (Val1) Val1 = Builder.CreateBitCast(Val1, IPtrTy); 751fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (Val2) Val2 = Builder.CreateBitCast(Val2, IPtrTy); 752fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (Dest && !E->isCmpXChg()) Dest = Builder.CreateBitCast(Dest, IPtrTy); 753fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 754fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (isa<llvm::ConstantInt>(Order)) { 755fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 756fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall switch (ord) { 757651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case AtomicExpr::AO_ABI_memory_order_relaxed: 758ef8225444452a1486bd721f3285301fe84643b00Stephen Hines EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, 759651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Size, Align, llvm::Monotonic); 760fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 761651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case AtomicExpr::AO_ABI_memory_order_consume: 762651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case AtomicExpr::AO_ABI_memory_order_acquire: 763fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (IsStore) 764fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; // Avoid crashing on code with undefined behavior 765ef8225444452a1486bd721f3285301fe84643b00Stephen Hines EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, 766651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Size, Align, llvm::Acquire); 767fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 768651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case AtomicExpr::AO_ABI_memory_order_release: 769fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (IsLoad) 770fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; // Avoid crashing on code with undefined behavior 771ef8225444452a1486bd721f3285301fe84643b00Stephen Hines EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, 772651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Size, Align, llvm::Release); 773fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 774651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case AtomicExpr::AO_ABI_memory_order_acq_rel: 775fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (IsLoad || IsStore) 776fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; // Avoid crashing on code with undefined behavior 777ef8225444452a1486bd721f3285301fe84643b00Stephen Hines EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, 778651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Size, Align, llvm::AcquireRelease); 779fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 780651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines case AtomicExpr::AO_ABI_memory_order_seq_cst: 781ef8225444452a1486bd721f3285301fe84643b00Stephen Hines EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, 782651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Size, Align, llvm::SequentiallyConsistent); 783fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 784fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall default: // invalid order 785fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // We should not ever get here normally, but it's hard to 786fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // enforce that in general. 787fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 788fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 789fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (E->getType()->isVoidType()) 7906bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 7914ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky return convertTempToRValue(OrigDest, E->getType(), E->getExprLoc()); 792fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 793fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 794fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Long case, when Order isn't obviously constant. 795fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 796fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Create all the relevant BB's 7976bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, 7986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines *ReleaseBB = nullptr, *AcqRelBB = nullptr, 7996bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines *SeqCstBB = nullptr; 800fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall MonotonicBB = createBasicBlock("monotonic", CurFn); 801fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (!IsStore) 802fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall AcquireBB = createBasicBlock("acquire", CurFn); 803fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (!IsLoad) 804fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall ReleaseBB = createBasicBlock("release", CurFn); 805fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (!IsLoad && !IsStore) 806fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall AcqRelBB = createBasicBlock("acqrel", CurFn); 807fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall SeqCstBB = createBasicBlock("seqcst", CurFn); 808fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 809fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 810fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Create the switch for the split 811fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // MonotonicBB is arbitrarily chosen as the default case; in practice, this 812fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // doesn't matter unless someone is crazy enough to use something that 813fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // doesn't fold to a constant for the ordering. 814fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 815fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB); 816fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 817fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Emit all the different atomics 818fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.SetInsertPoint(MonotonicBB); 819ef8225444452a1486bd721f3285301fe84643b00Stephen Hines EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, 820651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Size, Align, llvm::Monotonic); 821fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.CreateBr(ContBB); 822fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (!IsStore) { 823fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.SetInsertPoint(AcquireBB); 824ef8225444452a1486bd721f3285301fe84643b00Stephen Hines EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, 825651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Size, Align, llvm::Acquire); 826fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.CreateBr(ContBB); 827651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume), 828651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AcquireBB); 829651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acquire), 830651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AcquireBB); 831fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 832fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (!IsLoad) { 833fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.SetInsertPoint(ReleaseBB); 834ef8225444452a1486bd721f3285301fe84643b00Stephen Hines EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, 835651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Size, Align, llvm::Release); 836fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.CreateBr(ContBB); 837651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_release), 838651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ReleaseBB); 839fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 840fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (!IsLoad && !IsStore) { 841fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.SetInsertPoint(AcqRelBB); 842ef8225444452a1486bd721f3285301fe84643b00Stephen Hines EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, 843651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Size, Align, llvm::AcquireRelease); 844fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.CreateBr(ContBB); 845651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acq_rel), 846651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AcqRelBB); 847fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 848fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.SetInsertPoint(SeqCstBB); 849ef8225444452a1486bd721f3285301fe84643b00Stephen Hines EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, 850651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Size, Align, llvm::SequentiallyConsistent); 851fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.CreateBr(ContBB); 852651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst), 853651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SeqCstBB); 854fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 855fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Cleanup and return 856fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.SetInsertPoint(ContBB); 857fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (E->getType()->isVoidType()) 8586bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 8594ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky return convertTempToRValue(OrigDest, E->getType(), E->getExprLoc()); 860fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall} 8619eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 8629eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallllvm::Value *AtomicInfo::emitCastToAtomicIntPointer(llvm::Value *addr) const { 8639eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall unsigned addrspace = 8649eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall cast<llvm::PointerType>(addr->getType())->getAddressSpace(); 8659eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::IntegerType *ty = 8669eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits); 8679eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace)); 8689eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 8699eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 8709eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallRValue AtomicInfo::convertTempToRValue(llvm::Value *addr, 8714ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky AggValueSlot resultSlot, 8724ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky SourceLocation loc) const { 873336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman if (EvaluationKind == TEK_Aggregate) 874336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman return resultSlot.asRValue(); 8759eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 8769eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Drill into the padding structure if we have one. 8779eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (hasPadding()) 8789eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall addr = CGF.Builder.CreateStructGEP(addr, 0); 8799eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 8809eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Otherwise, just convert the temporary to an r-value using the 8819eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // normal conversion routine. 8824ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky return CGF.convertTempToRValue(addr, getValueType(), loc); 8839eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 8849eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 8859eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// Emit a load from an l-value of atomic type. Note that the r-value 8869eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// we produce is an r-value of the atomic *value* type. 8874ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick LewyckyRValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc, 8884ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky AggValueSlot resultSlot) { 8899eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AtomicInfo atomics(*this, src); 8909eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 8919eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Check whether we should use a library call. 8929eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (atomics.shouldUseLibcall()) { 8939eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *tempAddr; 894336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman if (!resultSlot.isIgnored()) { 8959eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall assert(atomics.getEvaluationKind() == TEK_Aggregate); 8969eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall tempAddr = resultSlot.getAddr(); 8979eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } else { 8989eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall tempAddr = CreateMemTemp(atomics.getAtomicType(), "atomic-load-temp"); 8999eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 9009eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 9019eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // void __atomic_load(size_t size, void *mem, void *return, int order); 9029eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CallArgList args; 9039eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall args.add(RValue::get(atomics.getAtomicSizeValue()), 9049eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall getContext().getSizeType()); 9059eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall args.add(RValue::get(EmitCastToVoidPtr(src.getAddress())), 9069eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall getContext().VoidPtrTy); 9079eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall args.add(RValue::get(EmitCastToVoidPtr(tempAddr)), 9089eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall getContext().VoidPtrTy); 909651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines args.add(RValue::get(llvm::ConstantInt::get( 910651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines IntTy, AtomicExpr::AO_ABI_memory_order_seq_cst)), 9119eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall getContext().IntTy); 9129eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall emitAtomicLibcall(*this, "__atomic_load", getContext().VoidTy, args); 9139eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 9149eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Produce the r-value. 9154ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky return atomics.convertTempToRValue(tempAddr, resultSlot, loc); 9169eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 9179eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 9189eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Okay, we're doing this natively. 9199eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *addr = atomics.emitCastToAtomicIntPointer(src.getAddress()); 9209eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::LoadInst *load = Builder.CreateLoad(addr, "atomic-load"); 9219eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall load->setAtomic(llvm::SequentiallyConsistent); 9229eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 9239eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Other decoration. 9249eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall load->setAlignment(src.getAlignment().getQuantity()); 9259eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (src.isVolatileQualified()) 9269eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall load->setVolatile(true); 9279eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (src.getTBAAInfo()) 9289eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CGM.DecorateInstruction(load, src.getTBAAInfo()); 9299eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 9309eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Okay, turn that back into the original value type. 9319eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall QualType valueType = atomics.getValueType(); 9329eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *result = load; 9339eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 9349eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // If we're ignoring an aggregate return, don't do anything. 9359eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (atomics.getEvaluationKind() == TEK_Aggregate && resultSlot.isIgnored()) 9366bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::getAggregate(nullptr, false); 9379eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 9389eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // The easiest way to do this this is to go through memory, but we 9399eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // try not to in some easy cases. 9409eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (atomics.getEvaluationKind() == TEK_Scalar && !atomics.hasPadding()) { 9419eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Type *resultTy = CGM.getTypes().ConvertTypeForMem(valueType); 9429eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (isa<llvm::IntegerType>(resultTy)) { 9439eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall assert(result->getType() == resultTy); 9449eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall result = EmitFromMemory(result, valueType); 9459eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } else if (isa<llvm::PointerType>(resultTy)) { 9469eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall result = Builder.CreateIntToPtr(result, resultTy); 9479eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } else { 9489eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall result = Builder.CreateBitCast(result, resultTy); 9499eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 9509eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return RValue::get(result); 9519eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 9529eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 9539eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Create a temporary. This needs to be big enough to hold the 9549eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // atomic integer. 9559eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *temp; 9569eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall bool tempIsVolatile = false; 9579eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CharUnits tempAlignment; 958336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman if (atomics.getEvaluationKind() == TEK_Aggregate) { 9599eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall assert(!resultSlot.isIgnored()); 960336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman temp = resultSlot.getAddr(); 961336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman tempAlignment = atomics.getValueAlignment(); 9629eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall tempIsVolatile = resultSlot.isVolatile(); 9639eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } else { 9649eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall temp = CreateMemTemp(atomics.getAtomicType(), "atomic-load-temp"); 9659eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall tempAlignment = atomics.getAtomicAlignment(); 9669eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 9679eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 9689eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Slam the integer into the temporary. 9699eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *castTemp = atomics.emitCastToAtomicIntPointer(temp); 9709eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall Builder.CreateAlignedStore(result, castTemp, tempAlignment.getQuantity()) 9719eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall ->setVolatile(tempIsVolatile); 9729eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 9734ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky return atomics.convertTempToRValue(temp, resultSlot, loc); 9749eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 9759eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 9769eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 9779eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 9789eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// Copy an r-value into memory as part of storing to an atomic type. 9799eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// This needs to create a bit-pattern suitable for atomic operations. 9809eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallvoid AtomicInfo::emitCopyIntoMemory(RValue rvalue, LValue dest) const { 9819eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // If we have an r-value, the rvalue should be of the atomic type, 9829eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // which means that the caller is responsible for having zeroed 9839eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // any padding. Just do an aggregate copy of that type. 9849eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (rvalue.isAggregate()) { 9859eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CGF.EmitAggregateCopy(dest.getAddress(), 9869eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall rvalue.getAggregateAddr(), 9879eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall getAtomicType(), 9889eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall (rvalue.isVolatileQualified() 9899eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall || dest.isVolatileQualified()), 9909eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall dest.getAlignment()); 9919eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return; 9929eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 9939eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 9949eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Okay, otherwise we're copying stuff. 9959eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 9969eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Zero out the buffer if necessary. 9979eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall emitMemSetZeroIfNecessary(dest); 9989eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 9999eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Drill past the padding if present. 10009eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall dest = projectValue(dest); 10019eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 10029eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Okay, store the rvalue in. 10039eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (rvalue.isScalar()) { 10049eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CGF.EmitStoreOfScalar(rvalue.getScalarVal(), dest, /*init*/ true); 10059eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } else { 10069eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CGF.EmitStoreOfComplex(rvalue.getComplexVal(), dest, /*init*/ true); 10079eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 10089eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 10099eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 10109eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 10119eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// Materialize an r-value into memory for the purposes of storing it 10129eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// to an atomic type. 10139eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallllvm::Value *AtomicInfo::materializeRValue(RValue rvalue) const { 10149eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Aggregate r-values are already in memory, and EmitAtomicStore 10159eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // requires them to be values of the atomic type. 10169eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (rvalue.isAggregate()) 10179eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return rvalue.getAggregateAddr(); 10189eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 10199eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Otherwise, make a temporary and materialize into it. 10209eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *temp = CGF.CreateMemTemp(getAtomicType(), "atomic-store-temp"); 10219eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall LValue tempLV = CGF.MakeAddrLValue(temp, getAtomicType(), getAtomicAlignment()); 10229eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall emitCopyIntoMemory(rvalue, tempLV); 10239eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return temp; 10249eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 10259eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 10269eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// Emit a store to an l-value of atomic type. 10279eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// 10289eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// Note that the r-value is expected to be an r-value *of the atomic 10299eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// type*; this means that for aggregate r-values, it should include 10309eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// storage for any padding that was necessary. 10315d4a7559df106959dd721744c8971547f0f09097Nick Lewyckyvoid CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, bool isInit) { 10329eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // If this is an aggregate r-value, it should agree in type except 10339eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // maybe for address-space qualification. 10349eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall assert(!rvalue.isAggregate() || 10359eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall rvalue.getAggregateAddr()->getType()->getPointerElementType() 10369eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall == dest.getAddress()->getType()->getPointerElementType()); 10379eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 10389eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AtomicInfo atomics(*this, dest); 10399eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 10409eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // If this is an initialization, just put the value there normally. 10419eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (isInit) { 10429eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall atomics.emitCopyIntoMemory(rvalue, dest); 10439eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return; 10449eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 10459eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 10469eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Check whether we should use a library call. 10479eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (atomics.shouldUseLibcall()) { 10489eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Produce a source address. 10499eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *srcAddr = atomics.materializeRValue(rvalue); 10509eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 10519eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // void __atomic_store(size_t size, void *mem, void *val, int order) 10529eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CallArgList args; 10539eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall args.add(RValue::get(atomics.getAtomicSizeValue()), 10549eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall getContext().getSizeType()); 10559eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall args.add(RValue::get(EmitCastToVoidPtr(dest.getAddress())), 10569eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall getContext().VoidPtrTy); 10579eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall args.add(RValue::get(EmitCastToVoidPtr(srcAddr)), 10589eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall getContext().VoidPtrTy); 1059651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines args.add(RValue::get(llvm::ConstantInt::get( 1060651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines IntTy, AtomicExpr::AO_ABI_memory_order_seq_cst)), 10619eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall getContext().IntTy); 10629eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args); 10639eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return; 10649eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 10659eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 10669eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Okay, we're doing this natively. 10679eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *intValue; 10689eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 10699eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // If we've got a scalar value of the right size, try to avoid going 10709eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // through memory. 10719eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (rvalue.isScalar() && !atomics.hasPadding()) { 10729eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *value = rvalue.getScalarVal(); 10739eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (isa<llvm::IntegerType>(value->getType())) { 10749eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall intValue = value; 10759eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } else { 10769eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::IntegerType *inputIntTy = 10779eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::IntegerType::get(getLLVMContext(), atomics.getValueSizeInBits()); 10789eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (isa<llvm::PointerType>(value->getType())) { 10799eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall intValue = Builder.CreatePtrToInt(value, inputIntTy); 10809eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } else { 10819eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall intValue = Builder.CreateBitCast(value, inputIntTy); 10829eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 10839eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 10849eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 10859eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Otherwise, we need to go through memory. 10869eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } else { 10879eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Put the r-value in memory. 10889eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *addr = atomics.materializeRValue(rvalue); 10899eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 10909eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Cast the temporary to the atomic int type and pull a value out. 10919eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall addr = atomics.emitCastToAtomicIntPointer(addr); 10929eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall intValue = Builder.CreateAlignedLoad(addr, 10939eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall atomics.getAtomicAlignment().getQuantity()); 10949eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 10959eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 10969eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Do the atomic store. 10979eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *addr = atomics.emitCastToAtomicIntPointer(dest.getAddress()); 10989eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::StoreInst *store = Builder.CreateStore(intValue, addr); 10999eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 11009eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Initializations don't need to be atomic. 11019eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (!isInit) store->setAtomic(llvm::SequentiallyConsistent); 11029eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 11039eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Other decoration. 11049eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall store->setAlignment(dest.getAlignment().getQuantity()); 11059eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (dest.isVolatileQualified()) 11069eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall store->setVolatile(true); 11079eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (dest.getTBAAInfo()) 11089eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CGM.DecorateInstruction(store, dest.getTBAAInfo()); 11099eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 11109eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 11119eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallvoid CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { 11129eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AtomicInfo atomics(*this, dest); 11139eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 11149eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall switch (atomics.getEvaluationKind()) { 11159eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall case TEK_Scalar: { 11169eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *value = EmitScalarExpr(init); 11179eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall atomics.emitCopyIntoMemory(RValue::get(value), dest); 11189eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return; 11199eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 11209eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 11219eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall case TEK_Complex: { 11229eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall ComplexPairTy value = EmitComplexExpr(init); 11239eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall atomics.emitCopyIntoMemory(RValue::getComplex(value), dest); 11249eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return; 11259eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 11269eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 11279eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall case TEK_Aggregate: { 1128336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman // Fix up the destination if the initializer isn't an expression 1129336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman // of atomic type. 1130336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman bool Zeroed = false; 11319eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (!init->getType()->isAtomicType()) { 1132336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman Zeroed = atomics.emitMemSetZeroIfNecessary(dest); 11339eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall dest = atomics.projectValue(dest); 11349eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 11359eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 11369eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Evaluate the expression directly into the destination. 11379eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AggValueSlot slot = AggValueSlot::forLValue(dest, 11389eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AggValueSlot::IsNotDestructed, 11399eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AggValueSlot::DoesNotNeedGCBarriers, 1140336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman AggValueSlot::IsNotAliased, 1141336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman Zeroed ? AggValueSlot::IsZeroed : 1142336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman AggValueSlot::IsNotZeroed); 1143336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman 11449eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall EmitAggExpr(init, slot); 11459eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return; 11469eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 11479eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 11489eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm_unreachable("bad evaluation kind"); 11499eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 1150